mirror of https://github.com/pikami/tiktok-dl.git
Embed scraper into compiled binary, extract strings to resources file
This commit is contained in:
Pijus Kamandulis
parent
70c3faf17e
commit
ea2b866f9a
|
|
@ -7,3 +7,4 @@ tiktok-dl
|
|||
batch_file.txt
|
||||
debug.log
|
||||
.scannerwork
|
||||
scraper.min.js
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"github.com/chromedp/chromedp"
|
||||
|
||||
config "github.com/pikami/tiktok-dl/models/config"
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
utils "github.com/pikami/tiktok-dl/utils"
|
||||
log "github.com/pikami/tiktok-dl/utils/log"
|
||||
)
|
||||
|
|
@ -54,7 +55,7 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, e
|
|||
// Navigate to user's page
|
||||
chromedp.Navigate(url),
|
||||
// Execute url grabber script
|
||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
||||
chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
|
||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||
// Wait until custom js finishes
|
||||
chromedp.WaitVisible(`video_urls`),
|
||||
|
|
@ -73,7 +74,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
|||
// Navigate to user's page
|
||||
chromedp.Navigate(url),
|
||||
// Execute url grabber script
|
||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
||||
chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
|
||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||
); err != nil {
|
||||
return "", err
|
||||
|
|
@ -85,9 +86,9 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
|||
}
|
||||
|
||||
if jsOutput != "0" {
|
||||
log.Logf("\rPreloading... %s items have been found.", jsOutput)
|
||||
log.Logf(res.PreloadingItemsFound, jsOutput)
|
||||
} else {
|
||||
log.Logf("\rPreloading...")
|
||||
log.Logf(res.Preloading)
|
||||
}
|
||||
|
||||
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
|
||||
|
|
@ -101,7 +102,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
|||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
log.Log("\nRetrieving items...")
|
||||
log.Log(res.Retrieving)
|
||||
if err := chromedp.Run(ctx,
|
||||
// Wait until custom js finishes
|
||||
chromedp.WaitVisible(`video_urls`),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,64 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
)
|
||||
|
||||
type resource struct {
|
||||
Package string
|
||||
FileName string
|
||||
Values map[string]string
|
||||
}
|
||||
|
||||
func (r resource) generate() {
|
||||
filename := fmt.Sprintf("%s/%s", outputDir, r.FileName)
|
||||
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
checkErr.CheckErr(err)
|
||||
defer f.Close()
|
||||
|
||||
// Header
|
||||
header := fmt.Sprintf("// Package %s - This file is automatically generated.\n"+
|
||||
"// Do not edit this file manually.\n"+
|
||||
"// Check `/generator/resources.go` to change generated content\n"+
|
||||
"package %s\n", r.Package, r.Package)
|
||||
|
||||
if _, err := f.WriteString(header); err != nil {
|
||||
checkErr.CheckErr(err)
|
||||
}
|
||||
|
||||
// Values
|
||||
for key, value := range r.Values {
|
||||
value = strings.ReplaceAll(value, "\n", "\\n")
|
||||
value = strings.ReplaceAll(value, "\r", "\\r")
|
||||
valueLine := fmt.Sprintf("\n//%s -\nvar %s = \"%s\"\n", key, key, value)
|
||||
if _, err := f.WriteString(valueLine); err != nil {
|
||||
checkErr.CheckErr(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fileContentsOrDefault(file string) string {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
fmt.Printf("Failed to load file: %s\n", file)
|
||||
}
|
||||
}()
|
||||
|
||||
return safeString(fileio.ReadFileToString(file))
|
||||
}
|
||||
|
||||
func safeString(str string) string {
|
||||
escaped := strings.ReplaceAll(str, "\"", "\\\"")
|
||||
return strings.ReplaceAll(escaped, "\n", "")
|
||||
}
|
||||
|
||||
func main() {
|
||||
for _, i := range res {
|
||||
i.generate()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
package main
|
||||
|
||||
var (
|
||||
outputDir = "../resources"
|
||||
|
||||
res = []resource{
|
||||
resource{
|
||||
Package: "resources",
|
||||
FileName: "scraper.go",
|
||||
Values: map[string]string{
|
||||
"ScraperPath": "scraper.js",
|
||||
"ScraperScript": fileContentsOrDefault("../scraper.min.js"),
|
||||
},
|
||||
},
|
||||
resource{
|
||||
Package: "resources",
|
||||
FileName: "errorStrings.go",
|
||||
Values: map[string]string{
|
||||
"ErrorCouldNotSerializeJSON": "Could not serialize json for video: %s\n",
|
||||
"ErrorCouldNotRecogniseURL": "Could not recognise URL format of string %s",
|
||||
"Error": "Error : %s\n",
|
||||
"ErrorPathNotFound": "File path %s not found.",
|
||||
"FailedOnItem": "Failed while scraping item: %s\n",
|
||||
"FailedToLoadScraper": "Failed to load scraper",
|
||||
},
|
||||
},
|
||||
resource{
|
||||
Package: "resources",
|
||||
FileName: "messages.go",
|
||||
Values: map[string]string{
|
||||
"PreloadingItemsFound": "\rPreloading... %s items have been found.",
|
||||
"Preloading": "\rPreloading...",
|
||||
"Retrieving": "\nRetrieving items...",
|
||||
"ItemsFoundInArchive": "%d items, found in archive. Skipping...\n",
|
||||
"Downloaded": "\r[%d/%d] Downloaded",
|
||||
"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
|
||||
" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt",
|
||||
},
|
||||
},
|
||||
resource{
|
||||
Package: "resources",
|
||||
FileName: "flags.go",
|
||||
Values: map[string]string{
|
||||
// Output
|
||||
"OutputFlag": "output",
|
||||
"OutputDefault": "./downloads",
|
||||
"OutputDescription": "Output path",
|
||||
// Batch file
|
||||
"BatchFlag": "batch-file",
|
||||
"BatchDefault": "",
|
||||
"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
|
||||
// Archive
|
||||
"ArchiveFlag": "archive",
|
||||
"ArchiveDefault": "",
|
||||
"ArchiveDescription": "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.",
|
||||
// Fail log
|
||||
"FailLogFlag": "fail-log",
|
||||
"FailLogDefault": "",
|
||||
"FailLogDescription": "Write failed items to log file",
|
||||
// Debug
|
||||
"DebugFlag": "debug",
|
||||
"DebugDefault": "false",
|
||||
"DebugDescription": "Enables debug mode",
|
||||
// Metadata
|
||||
"MetadataFlag": "metadata",
|
||||
"MetadataDefault": "false",
|
||||
"MetadataDescription": "Write video metadata to a .json file",
|
||||
// Quiet
|
||||
"QuietFlag": "quiet",
|
||||
"QuietDefault": "false",
|
||||
"QuietDescription": "Suppress output",
|
||||
// JSON only
|
||||
"JsonFlag": "json",
|
||||
"JsonDefault": "false",
|
||||
"JsonDescription": "Just get JSON data from scraper (without video downloading)",
|
||||
// Deadline
|
||||
"DeadlineFlag": "deadline",
|
||||
"DeadlineDefault": "1500",
|
||||
"DeadlineDescription": "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)",
|
||||
// Limit
|
||||
"LimitFlag": "limit",
|
||||
"LimitDefault": "0",
|
||||
"LimitDescription": "Sets the videos count limit (useful when there too many videos from the user or by hashtag)",
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
|
@ -4,6 +4,9 @@ import (
|
|||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
)
|
||||
|
||||
// Config - Runtime configuration
|
||||
|
|
@ -23,22 +26,21 @@ var Config struct {
|
|||
|
||||
// GetConfig - Returns Config object
|
||||
func GetConfig() {
|
||||
outputPath := flag.String("output", "./downloads", "Output path")
|
||||
batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
|
||||
archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.")
|
||||
failLogPath := flag.String("fail-log", "", "Write failed items to log file")
|
||||
debug := flag.Bool("debug", false, "Enables debug mode")
|
||||
metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
|
||||
quiet := flag.Bool("quiet", false, "Suppress output")
|
||||
jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
|
||||
deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
|
||||
limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
|
||||
outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription)
|
||||
batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription)
|
||||
archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription)
|
||||
failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription)
|
||||
debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription)
|
||||
metadata := flag.Bool(res.MetadataFlag, parseBool(res.MetadataDefault), res.MetadataDescription)
|
||||
quiet := flag.Bool(res.QuietFlag, parseBool(res.QuietDefault), res.QuietDescription)
|
||||
jsonOnly := flag.Bool(res.JsonFlag, parseBool(res.JsonDefault), res.JsonDescription)
|
||||
deadline := flag.Int(res.DeadlineFlag, parseInt(res.DeadlineDefault), res.DeadlineDescription)
|
||||
limit := flag.Int(res.LimitFlag, parseInt(res.LimitDefault), res.LimitDescription)
|
||||
flag.Parse()
|
||||
|
||||
args := flag.Args()
|
||||
if len(args) < 1 && *batchFilePath == "" {
|
||||
fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
|
||||
fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
|
||||
fmt.Println(res.UsageLine)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
|
|
@ -61,3 +63,19 @@ func GetConfig() {
|
|||
Config.Deadline = *deadline
|
||||
Config.Limit = *limit
|
||||
}
|
||||
|
||||
func parseBool(str string) bool {
|
||||
val, err := strconv.ParseBool(str)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func parseInt(str string) int {
|
||||
val, err := strconv.Atoi(str)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,15 @@
|
|||
{
|
||||
"name": "tiktok-dl",
|
||||
"version": "0.0.1",
|
||||
"version": "2.0.0",
|
||||
"scripts": {
|
||||
"install-dependencies": "go get -v -t -d ./...",
|
||||
"test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
|
||||
"test": "go test -v ./models && go test -v ./utils",
|
||||
"clean": "rm -rf out",
|
||||
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
|
||||
"clean": "rm -rf out && rm -f scraper.min.js",
|
||||
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > scraper.min.js",
|
||||
"build:res": "cd generator && go run . && cd ..",
|
||||
"build:app": "go build -o out/tiktok-dl -v .",
|
||||
"build:dist": "mkdir out && npm run build:app && npm run build:scraper",
|
||||
"build:dist": "npm run clean && mkdir out && npm run build:scraper && npm run build:res && npm run build:app",
|
||||
"build": "go build -v .",
|
||||
"sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
// Package resources - This file is automatically generated.
|
||||
// Do not edit this file manually.
|
||||
// Check `/generator/resources.go` to change generated content
|
||||
package resources
|
||||
|
||||
//ErrorCouldNotSerializeJSON -
|
||||
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
||||
|
||||
//ErrorCouldNotRecogniseURL -
|
||||
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
||||
|
||||
//Error -
|
||||
var Error = "Error : %s\n"
|
||||
|
||||
//ErrorPathNotFound -
|
||||
var ErrorPathNotFound = "File path %s not found."
|
||||
|
||||
//FailedOnItem -
|
||||
var FailedOnItem = "Failed while scraping item: %s\n"
|
||||
|
||||
//FailedToLoadScraper -
|
||||
var FailedToLoadScraper = "Failed to load scraper"
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
// Package resources - This file is automatically generated.
|
||||
// Do not edit this file manually.
|
||||
// Check `/generator/resources.go` to change generated content
|
||||
package resources
|
||||
|
||||
//BatchDescription -
|
||||
var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored."
|
||||
|
||||
//LimitDefault -
|
||||
var LimitDefault = "0"
|
||||
|
||||
//DebugDescription -
|
||||
var DebugDescription = "Enables debug mode"
|
||||
|
||||
//QuietDescription -
|
||||
var QuietDescription = "Suppress output"
|
||||
|
||||
//MetadataDefault -
|
||||
var MetadataDefault = "false"
|
||||
|
||||
//LimitDescription -
|
||||
var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
|
||||
|
||||
//OutputDescription -
|
||||
var OutputDescription = "Output path"
|
||||
|
||||
//BatchDefault -
|
||||
var BatchDefault = ""
|
||||
|
||||
//FailLogDefault -
|
||||
var FailLogDefault = ""
|
||||
|
||||
//MetadataFlag -
|
||||
var MetadataFlag = "metadata"
|
||||
|
||||
//ArchiveFlag -
|
||||
var ArchiveFlag = "archive"
|
||||
|
||||
//MetadataDescription -
|
||||
var MetadataDescription = "Write video metadata to a .json file"
|
||||
|
||||
//QuietDefault -
|
||||
var QuietDefault = "false"
|
||||
|
||||
//DeadlineDefault -
|
||||
var DeadlineDefault = "1500"
|
||||
|
||||
//JsonFlag -
|
||||
var JsonFlag = "json"
|
||||
|
||||
//JsonDefault -
|
||||
var JsonDefault = "false"
|
||||
|
||||
//DeadlineDescription -
|
||||
var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
|
||||
|
||||
//OutputFlag -
|
||||
var OutputFlag = "output"
|
||||
|
||||
//OutputDefault -
|
||||
var OutputDefault = "./downloads"
|
||||
|
||||
//FailLogFlag -
|
||||
var FailLogFlag = "fail-log"
|
||||
|
||||
//DebugDefault -
|
||||
var DebugDefault = "false"
|
||||
|
||||
//ArchiveDefault -
|
||||
var ArchiveDefault = ""
|
||||
|
||||
//FailLogDescription -
|
||||
var FailLogDescription = "Write failed items to log file"
|
||||
|
||||
//BatchFlag -
|
||||
var BatchFlag = "batch-file"
|
||||
|
||||
//QuietFlag -
|
||||
var QuietFlag = "quiet"
|
||||
|
||||
//DeadlineFlag -
|
||||
var DeadlineFlag = "deadline"
|
||||
|
||||
//LimitFlag -
|
||||
var LimitFlag = "limit"
|
||||
|
||||
//ArchiveDescription -
|
||||
var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it."
|
||||
|
||||
//DebugFlag -
|
||||
var DebugFlag = "debug"
|
||||
|
||||
//JsonDescription -
|
||||
var JsonDescription = "Just get JSON data from scraper (without video downloading)"
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
// Package resources - This file is automatically generated.
|
||||
// Do not edit this file manually.
|
||||
// Check `/generator/resources.go` to change generated content
|
||||
package resources
|
||||
|
||||
//PreloadingItemsFound -
|
||||
var PreloadingItemsFound = "\rPreloading... %s items have been found."
|
||||
|
||||
//Preloading -
|
||||
var Preloading = "\rPreloading..."
|
||||
|
||||
//Retrieving -
|
||||
var Retrieving = "\nRetrieving items..."
|
||||
|
||||
//ItemsFoundInArchive -
|
||||
var ItemsFoundInArchive = "%d items, found in archive. Skipping...\n"
|
||||
|
||||
//Downloaded -
|
||||
var Downloaded = "\r[%d/%d] Downloaded"
|
||||
|
||||
//UsageLine -
|
||||
var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt"
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
// Package resources - This file is automatically generated.
|
||||
// Do not edit this file manually.
|
||||
// Check `/generator/resources.go` to change generated content
|
||||
package resources
|
||||
|
||||
//ScraperScript -
|
||||
var ScraperScript = "optStrings={selectors:{feedLoading:\"div.tiktok-loading.feed-loading\",modalArrowRight:\"div.video-card-modal > div > img.arrow-right\",modalClose:\".video-card-modal > div > div.close\",modalPlayer:\"div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video\",modalShareInput:\".copy-link-container > input\",modalCaption:\"div.video-card-big > div.content-container > div.video-meta-info > h1\",modalSoundLink:\"div.content-container > div.video-meta-info > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > div > video\",videoShareInput:\"div.content-container.border > div.copy-link-container > input\",videoCaption:\"div.content-container.border > div.video-meta-info > h1\",videoSoundLink:\"div.content-container.border > div.video-meta-info > h2.music-info > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var i=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];i.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):i.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=document.querySelector(optStrings.selectors.modalShareInput).value,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),i=document.querySelector(optStrings.selectors.modalUploader).textContent,n=o.getAttribute(\"href\");return{url:e,shareLink:t,caption:r,uploader:i,sound:{title:o.text,link:n}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=document.querySelector(optStrings.selectors.videoShareInput).value,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),i=document.querySelector(optStrings.selectors.videoUploader).textContent,n=o.getAttribute(\"href\");return{url:e,shareLink:t,caption:r,uploader:i,sound:{title:o.text,link:n}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var i=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(i!==t.count)currentState.preloadCount=t.count;else{if(document.querySelector(optStrings.selectors.feedLoading))return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"
|
||||
|
||||
//ScraperPath -
|
||||
var ScraperPath = "scraper.js"
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
package resources
|
||||
|
||||
// ErrorCouldNotSerializeJSON -
|
||||
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
||||
|
||||
// ErrorCouldNotRecogniseURL -
|
||||
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
||||
|
||||
// Error -
|
||||
var Error = "Error : %s\n"
|
||||
|
||||
// ErrorPathNotFound -
|
||||
var ErrorPathNotFound = "File path %s not found."
|
||||
|
||||
// FailedOnItem -
|
||||
var FailedOnItem = "Failed while scraping item: %s\n"
|
||||
|
|
@ -3,6 +3,7 @@ package utils
|
|||
import (
|
||||
models "github.com/pikami/tiktok-dl/models"
|
||||
config "github.com/pikami/tiktok-dl/models/config"
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
log "github.com/pikami/tiktok-dl/utils/log"
|
||||
)
|
||||
|
|
@ -36,7 +37,7 @@ func RemoveArchivedItems(uploads []models.Upload) []models.Upload {
|
|||
|
||||
removedCount := lenBeforeRemoval - len(uploads)
|
||||
if removedCount > 0 {
|
||||
log.Logf("%d items, found in archive. Skipping...\n", removedCount)
|
||||
log.Logf(res.ItemsFoundInArchive, removedCount)
|
||||
}
|
||||
|
||||
return uploads
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
resources "github.com/pikami/tiktok-dl/resources"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
)
|
||||
|
||||
// GetScraper - Retrieve scraper
|
||||
func GetScraper() string {
|
||||
if fileio.CheckIfExists(resources.ScraperPath) {
|
||||
return ReadFileAsString(resources.ScraperPath)
|
||||
}
|
||||
|
||||
if resources.ScraperScript != "" {
|
||||
return resources.ScraperScript
|
||||
}
|
||||
|
||||
panic(resources.FailedToLoadScraper)
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import (
|
|||
|
||||
client "github.com/pikami/tiktok-dl/client"
|
||||
config "github.com/pikami/tiktok-dl/models/config"
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
utils "github.com/pikami/tiktok-dl/utils"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
log "github.com/pikami/tiktok-dl/utils/log"
|
||||
|
|
@ -35,7 +36,7 @@ func DownloadHashtag(url string) {
|
|||
|
||||
for index, upload := range uploads {
|
||||
downloadVideo(upload, downloadDir)
|
||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
||||
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||
}
|
||||
log.Log()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
|
||||
client "github.com/pikami/tiktok-dl/client"
|
||||
config "github.com/pikami/tiktok-dl/models/config"
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
utils "github.com/pikami/tiktok-dl/utils"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
log "github.com/pikami/tiktok-dl/utils/log"
|
||||
|
|
@ -34,7 +35,7 @@ func DownloadMusic(url string) {
|
|||
|
||||
fileio.InitOutputDirectory(downloadDir)
|
||||
downloadVideo(upload, downloadDir)
|
||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
||||
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||
}
|
||||
log.Log()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
client "github.com/pikami/tiktok-dl/client"
|
||||
config "github.com/pikami/tiktok-dl/models/config"
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
utils "github.com/pikami/tiktok-dl/utils"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
log "github.com/pikami/tiktok-dl/utils/log"
|
||||
|
|
@ -36,7 +37,7 @@ func DownloadUser(username string) {
|
|||
|
||||
for index, upload := range uploads {
|
||||
downloadVideo(upload, downloadDir)
|
||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
||||
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||
}
|
||||
log.Log()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
client "github.com/pikami/tiktok-dl/client"
|
||||
models "github.com/pikami/tiktok-dl/models"
|
||||
config "github.com/pikami/tiktok-dl/models/config"
|
||||
res "github.com/pikami/tiktok-dl/resources"
|
||||
utils "github.com/pikami/tiktok-dl/utils"
|
||||
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||
log "github.com/pikami/tiktok-dl/utils/log"
|
||||
|
|
@ -34,7 +35,7 @@ func DownloadSingleVideo(url string) {
|
|||
|
||||
fileio.InitOutputDirectory(downloadDir)
|
||||
downloadVideo(upload, downloadDir)
|
||||
log.Log("[1/1] Downloaded\n")
|
||||
log.Logf(res.Downloaded, 1, 1)
|
||||
}
|
||||
|
||||
// DownloadVideo - Downloads one video
|
||||
|
|
|
|||
Loading…
Reference in New Issue