mirror of
https://github.com/pikami/tiktok-dl.git
synced 2025-03-14 13:54:34 +00:00
Compare commits
11 Commits
Author | SHA1 | Date | |
---|---|---|---|
|
e2e4ba0d4b | ||
|
7a25f521cb | ||
|
e9715c0d40 | ||
|
a70dc4cf04 | ||
|
ea2b866f9a | ||
|
70c3faf17e | ||
|
0a08f62dfe | ||
|
bb6e924866 | ||
|
af59659f41 | ||
|
92006d864f | ||
|
668b050dee |
6
.github/workflows/go.yml
vendored
6
.github/workflows/go.yml
vendored
@ -34,6 +34,12 @@ jobs:
|
|||||||
- name: Build
|
- name: Build
|
||||||
run: npm run build:dist
|
run: npm run build:dist
|
||||||
|
|
||||||
|
- name: Copy license
|
||||||
|
run: cp LICENSE out
|
||||||
|
|
||||||
|
- name: Add execute permission
|
||||||
|
run: chmod +x out/tiktok-dl
|
||||||
|
|
||||||
- name: Upload Unix Artifacts
|
- name: Upload Unix Artifacts
|
||||||
if: startsWith(matrix.os, 'ubuntu-')
|
if: startsWith(matrix.os, 'ubuntu-')
|
||||||
uses: actions/upload-artifact@v1
|
uses: actions/upload-artifact@v1
|
||||||
|
2
.gitignore
vendored
2
.gitignore
vendored
@ -6,3 +6,5 @@ downloads
|
|||||||
tiktok-dl
|
tiktok-dl
|
||||||
batch_file.txt
|
batch_file.txt
|
||||||
debug.log
|
debug.log
|
||||||
|
.scannerwork
|
||||||
|
scraper.min.js
|
||||||
|
12
.sonar/sonar-project.properties
Normal file
12
.sonar/sonar-project.properties
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
sonar.organization=pikami
|
||||||
|
sonar.projectKey=tiktok-dl
|
||||||
|
|
||||||
|
sonar.host.url=https://sonarcloud.io
|
||||||
|
|
||||||
|
sonar.sources=.
|
||||||
|
sonar.exclusions=**/*_test.go,**/node_modules/**
|
||||||
|
|
||||||
|
sonar.tests=.
|
||||||
|
sonar.test.inclusions=**/*_test.go
|
||||||
|
sonar.test.exclusions=**/node_modules/**
|
||||||
|
sonar.go.coverage.reportPaths=cov.out
|
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2020 pikami
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
36
README.md
36
README.md
@ -1,30 +1,44 @@
|
|||||||
# TikTok-DL
|
# TikTok-DL
|
||||||
|
|
||||||
[](https://goreportcard.com/report/github.com/pikami/tiktok-dl)
|
[](https://goreportcard.com/report/github.com/pikami/tiktok-dl)
|
||||||

|
[](https://github.com/pikami/tiktok-dl/actions)
|
||||||
|
|
||||||
A simple tiktok video downloader written in go
|
A simple tiktok video downloader written in go
|
||||||
|
|
||||||
## Basic usage
|
```diff
|
||||||
Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
|
- This tool is not working currenly, I will revive it in the future when I have some free time
|
||||||
You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\
|
```
|
||||||
You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\
|
|
||||||
You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\
|
|
||||||
You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt`
|
|
||||||
|
|
||||||
## Build instructions
|
## Basic usage examples
|
||||||
Clone this repository and run `go build` to build the executable.
|
Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
|
||||||
|
You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\
|
||||||
|
You can download single video by running `./tiktok-dl VIDEO_URL`\
|
||||||
|
You can download items listed in a text file by running `./tiktok-dl -batch-file path/to/items.txt`
|
||||||
|
|
||||||
|
## Usage Manual
|
||||||
|
```
|
||||||
|
Usage: tiktok-dl [OPTION]... TARGET
|
||||||
|
or: tiktok-dl [OPTION]... -batch-file BATCH_FILE
|
||||||
|
|
||||||
|
In the 1st form, download given `TARGET`.
|
||||||
|
In the 2nd form, download all targets listed in given `BATCH_FILE`.
|
||||||
|
```
|
||||||
|
|
||||||
## Available options
|
## Available options
|
||||||
* `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
|
* `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
|
||||||
* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
|
* `-batch-file some_file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
|
||||||
* `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
|
* `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
|
||||||
* `-debug` - enables debug mode
|
* `-debug` - enables debug mode
|
||||||
|
* `-fail-log some_file` - Write failed items to log file
|
||||||
* `-json` - Returns whole data, that was scraped from TikTok, in json
|
* `-json` - Returns whole data, that was scraped from TikTok, in json
|
||||||
* `-limit` - Sets the max count of video that will be downloaded (default infinity)
|
* `-limit` - Sets the max count of video that will be downloaded (default infinity)
|
||||||
* `-metadata` - Write video metadata to a .json file
|
* `-metadata` - Write video metadata to a .json file
|
||||||
* `-output some_directory` - Output path (default "./downloads")
|
* `-output some_directory` - Output path (default "./downloads")
|
||||||
* `-quiet` - Supress output
|
* `-quiet` - Suppress output
|
||||||
|
|
||||||
|
## Build instructions
|
||||||
|
1. Clone this repository
|
||||||
|
2. Run `go build` to build the executable.
|
||||||
|
|
||||||
## Acknowledgments
|
## Acknowledgments
|
||||||
This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \
|
This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \
|
||||||
|
@ -10,9 +10,10 @@ import (
|
|||||||
|
|
||||||
"github.com/chromedp/chromedp"
|
"github.com/chromedp/chromedp"
|
||||||
|
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
utils "../utils"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
log "../utils/log"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetMusicUploads - Get all uploads by given music
|
// GetMusicUploads - Get all uploads by given music
|
||||||
@ -54,7 +55,7 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, e
|
|||||||
// Navigate to user's page
|
// Navigate to user's page
|
||||||
chromedp.Navigate(url),
|
chromedp.Navigate(url),
|
||||||
// Execute url grabber script
|
// Execute url grabber script
|
||||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
|
||||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||||
// Wait until custom js finishes
|
// Wait until custom js finishes
|
||||||
chromedp.WaitVisible(`video_urls`),
|
chromedp.WaitVisible(`video_urls`),
|
||||||
@ -73,7 +74,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
|||||||
// Navigate to user's page
|
// Navigate to user's page
|
||||||
chromedp.Navigate(url),
|
chromedp.Navigate(url),
|
||||||
// Execute url grabber script
|
// Execute url grabber script
|
||||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
|
||||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
@ -85,9 +86,9 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
|||||||
}
|
}
|
||||||
|
|
||||||
if jsOutput != "0" {
|
if jsOutput != "0" {
|
||||||
log.Logf("\rPreloading... %s items have been found.", jsOutput)
|
log.Logf(res.PreloadingItemsFound, jsOutput)
|
||||||
} else {
|
} else {
|
||||||
log.Logf("\rPreloading...")
|
log.Logf(res.Preloading)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
|
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
|
||||||
@ -101,7 +102,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
|
|||||||
time.Sleep(50 * time.Millisecond)
|
time.Sleep(50 * time.Millisecond)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Log("\nRetrieving items...")
|
log.Log(res.Retrieving)
|
||||||
if err := chromedp.Run(ctx,
|
if err := chromedp.Run(ctx,
|
||||||
// Wait until custom js finishes
|
// Wait until custom js finishes
|
||||||
chromedp.WaitVisible(`video_urls`),
|
chromedp.WaitVisible(`video_urls`),
|
||||||
|
@ -3,8 +3,8 @@ package client
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
models "../models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetHashtagUploads - Get all uploads marked with given hashtag
|
// GetHashtagUploads - Get all uploads marked with given hashtag
|
||||||
|
@ -3,8 +3,8 @@ package client
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
models "../models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetMusicUploads - Get all uploads by given music
|
// GetMusicUploads - Get all uploads by given music
|
||||||
|
@ -2,16 +2,18 @@ package client
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"github.com/chromedp/chromedp"
|
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
config "../models/config"
|
"github.com/chromedp/chromedp"
|
||||||
log "../utils/log"
|
|
||||||
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
func GetRedirectUrl(url string) (string, error) {
|
// GetRedirectURL - Returns URL that the given URL redirects to
|
||||||
|
func GetRedirectURL(url string) (string, error) {
|
||||||
dir, err := ioutil.TempDir("", "chromedp-example")
|
dir, err := ioutil.TempDir("", "chromedp-example")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
|
@ -3,8 +3,8 @@ package client
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
models "../models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetUserUploads - Get all uploads by user
|
// GetUserUploads - Get all uploads by user
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
package client
|
package client
|
||||||
|
|
||||||
import (
|
import (
|
||||||
models "../models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetVideoDetails - returns details of video
|
// GetVideoDetails - returns details of video
|
||||||
|
64
generator/generator.go
Normal file
64
generator/generator.go
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||||
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
)
|
||||||
|
|
||||||
|
type resource struct {
|
||||||
|
Package string
|
||||||
|
FileName string
|
||||||
|
Values map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r resource) generate() {
|
||||||
|
filename := fmt.Sprintf("%s/%s", outputDir, r.FileName)
|
||||||
|
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||||
|
checkErr.CheckErr(err)
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
// Header
|
||||||
|
header := fmt.Sprintf("// Package %s - This file is automatically generated.\n"+
|
||||||
|
"// Do not edit this file manually.\n"+
|
||||||
|
"// Check `/generator/resources.go` to change generated content\n"+
|
||||||
|
"package %s\n", r.Package, r.Package)
|
||||||
|
|
||||||
|
if _, err := f.WriteString(header); err != nil {
|
||||||
|
checkErr.CheckErr(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Values
|
||||||
|
for key, value := range r.Values {
|
||||||
|
value = strings.ReplaceAll(value, "\n", "\\n")
|
||||||
|
value = strings.ReplaceAll(value, "\r", "\\r")
|
||||||
|
valueLine := fmt.Sprintf("\n//%s -\nvar %s = \"%s\"\n", key, key, value)
|
||||||
|
if _, err := f.WriteString(valueLine); err != nil {
|
||||||
|
checkErr.CheckErr(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileContentsOrDefault(file string) string {
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
fmt.Printf("Failed to load file: %s\n", file)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return safeString(fileio.ReadFileToString(file))
|
||||||
|
}
|
||||||
|
|
||||||
|
func safeString(str string) string {
|
||||||
|
escaped := strings.ReplaceAll(str, "\"", "\\\"")
|
||||||
|
return strings.ReplaceAll(escaped, "\n", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
for _, i := range res {
|
||||||
|
i.generate()
|
||||||
|
}
|
||||||
|
}
|
92
generator/resources.go
Normal file
92
generator/resources.go
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
var (
|
||||||
|
outputDir = "../resources"
|
||||||
|
|
||||||
|
res = []resource{
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "scraper.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
"ScraperPath": "scraper.js",
|
||||||
|
"ScraperScript": fileContentsOrDefault("../scraper.min.js"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "errorStrings.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
"ErrorCouldNotSerializeJSON": "Could not serialize json for video: %s\n",
|
||||||
|
"ErrorCouldNotRecogniseURL": "Could not recognise URL format of string %s",
|
||||||
|
"Error": "Error : %s\n",
|
||||||
|
"ErrorPathNotFound": "File path %s not found.",
|
||||||
|
"FailedOnItem": "Failed while scraping item: %s\n",
|
||||||
|
"FailedToLoadScraper": "Failed to load scraper",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "messages.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
"PreloadingItemsFound": "\rPreloading... %s items have been found.",
|
||||||
|
"Preloading": "\rPreloading...",
|
||||||
|
"Retrieving": "\nRetrieving items...",
|
||||||
|
"ItemsFoundInArchive": "%d items, found in archive. Skipping...\n",
|
||||||
|
"Downloaded": "\r[%d/%d] Downloaded",
|
||||||
|
"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
|
||||||
|
" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n" +
|
||||||
|
" or: tiktok-dl [OPTIONS] -scraped-data path/to/data.json",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource{
|
||||||
|
Package: "resources",
|
||||||
|
FileName: "flags.go",
|
||||||
|
Values: map[string]string{
|
||||||
|
// Output
|
||||||
|
"OutputFlag": "output",
|
||||||
|
"OutputDefault": "./downloads",
|
||||||
|
"OutputDescription": "Output path",
|
||||||
|
// Batch file
|
||||||
|
"BatchFlag": "batch-file",
|
||||||
|
"BatchDefault": "",
|
||||||
|
"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
|
||||||
|
// ScrapedData
|
||||||
|
"ScrapedDataFlag": "scraped-data",
|
||||||
|
"ScrapedDataDefault": "",
|
||||||
|
"ScrapedDataDescription": "Download videos from scrape file (json format)",
|
||||||
|
// Archive
|
||||||
|
"ArchiveFlag": "archive",
|
||||||
|
"ArchiveDefault": "",
|
||||||
|
"ArchiveDescription": "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.",
|
||||||
|
// Fail log
|
||||||
|
"FailLogFlag": "fail-log",
|
||||||
|
"FailLogDefault": "",
|
||||||
|
"FailLogDescription": "Write failed items to log file",
|
||||||
|
// Debug
|
||||||
|
"DebugFlag": "debug",
|
||||||
|
"DebugDefault": "false",
|
||||||
|
"DebugDescription": "Enables debug mode",
|
||||||
|
// Metadata
|
||||||
|
"MetadataFlag": "metadata",
|
||||||
|
"MetadataDefault": "false",
|
||||||
|
"MetadataDescription": "Write video metadata to a .json file",
|
||||||
|
// Quiet
|
||||||
|
"QuietFlag": "quiet",
|
||||||
|
"QuietDefault": "false",
|
||||||
|
"QuietDescription": "Suppress output",
|
||||||
|
// JSON only
|
||||||
|
"JsonFlag": "json",
|
||||||
|
"JsonDefault": "false",
|
||||||
|
"JsonDescription": "Just get JSON data from scraper (without video downloading)",
|
||||||
|
// Deadline
|
||||||
|
"DeadlineFlag": "deadline",
|
||||||
|
"DeadlineDefault": "1500",
|
||||||
|
"DeadlineDescription": "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)",
|
||||||
|
// Limit
|
||||||
|
"LimitFlag": "limit",
|
||||||
|
"LimitDefault": "0",
|
||||||
|
"LimitDescription": "Sets the videos count limit (useful when there too many videos from the user or by hashtag)",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
5
go.mod
Normal file
5
go.mod
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
module github.com/pikami/tiktok-dl
|
||||||
|
|
||||||
|
go 1.14
|
||||||
|
|
||||||
|
require github.com/chromedp/chromedp v0.5.3
|
16
go.sum
Normal file
16
go.sum
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac h1:T7V5BXqnYd55Hj/g5uhDYumg9Fp3rMTS6bykYtTIFX4=
|
||||||
|
github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g=
|
||||||
|
github.com/chromedp/chromedp v0.5.3 h1:F9LafxmYpsQhWQBdCs+6Sret1zzeeFyHS5LkRF//Ffg=
|
||||||
|
github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aaLxYgYj28w=
|
||||||
|
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
|
||||||
|
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||||
|
github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
|
||||||
|
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||||
|
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
|
||||||
|
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
|
||||||
|
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs=
|
||||||
|
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
|
||||||
|
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
|
||||||
|
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
|
||||||
|
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
|
||||||
|
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
11
main.go
11
main.go
@ -1,14 +1,15 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
config "./models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
workflows "./workflows"
|
workflows "github.com/pikami/tiktok-dl/workflows"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
config.GetConfig()
|
config.GetConfig()
|
||||||
url := config.Config.URL
|
url := config.Config.URL
|
||||||
batchFilePath := config.Config.BatchFilePath
|
batchFilePath := config.Config.BatchFilePath
|
||||||
|
scrapedDataFilePath := config.Config.ScrapedDataFilePath
|
||||||
|
|
||||||
// Batch file
|
// Batch file
|
||||||
if workflows.CanUseDownloadBatchFile(batchFilePath) {
|
if workflows.CanUseDownloadBatchFile(batchFilePath) {
|
||||||
@ -16,5 +17,11 @@ func main() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Scraped data file
|
||||||
|
if workflows.CanUseDownloadScrapedData(scrapedDataFilePath) {
|
||||||
|
workflows.DownloadScrapedData(scrapedDataFilePath)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
workflows.StartWorkflowByParameter(url)
|
workflows.StartWorkflowByParameter(url)
|
||||||
}
|
}
|
||||||
|
@ -4,6 +4,9 @@ import (
|
|||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config - Runtime configuration
|
// Config - Runtime configuration
|
||||||
@ -11,7 +14,9 @@ var Config struct {
|
|||||||
URL string
|
URL string
|
||||||
OutputPath string
|
OutputPath string
|
||||||
BatchFilePath string
|
BatchFilePath string
|
||||||
|
ScrapedDataFilePath string
|
||||||
ArchiveFilePath string
|
ArchiveFilePath string
|
||||||
|
FailLogFilePath string
|
||||||
Debug bool
|
Debug bool
|
||||||
MetaData bool
|
MetaData bool
|
||||||
Quiet bool
|
Quiet bool
|
||||||
@ -22,21 +27,22 @@ var Config struct {
|
|||||||
|
|
||||||
// GetConfig - Returns Config object
|
// GetConfig - Returns Config object
|
||||||
func GetConfig() {
|
func GetConfig() {
|
||||||
outputPath := flag.String("output", "./downloads", "Output path")
|
outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription)
|
||||||
batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
|
batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription)
|
||||||
archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.")
|
scrapedDataFilePath := flag.String(res.ScrapedDataFlag, res.ScrapedDataDefault, res.ScrapedDataDescription)
|
||||||
debug := flag.Bool("debug", false, "Enables debug mode")
|
archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription)
|
||||||
metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
|
failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription)
|
||||||
quiet := flag.Bool("quiet", false, "Supress output")
|
debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription)
|
||||||
jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
|
metadata := flag.Bool(res.MetadataFlag, parseBool(res.MetadataDefault), res.MetadataDescription)
|
||||||
deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
|
quiet := flag.Bool(res.QuietFlag, parseBool(res.QuietDefault), res.QuietDescription)
|
||||||
limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
|
jsonOnly := flag.Bool(res.JsonFlag, parseBool(res.JsonDefault), res.JsonDescription)
|
||||||
|
deadline := flag.Int(res.DeadlineFlag, parseInt(res.DeadlineDefault), res.DeadlineDescription)
|
||||||
|
limit := flag.Int(res.LimitFlag, parseInt(res.LimitDefault), res.LimitDescription)
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
args := flag.Args()
|
args := flag.Args()
|
||||||
if len(args) < 1 && *batchFilePath == "" {
|
if len(args) < 1 && *batchFilePath == "" && *scrapedDataFilePath == "" {
|
||||||
fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
|
fmt.Println(res.UsageLine)
|
||||||
fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
|
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -47,7 +53,9 @@ func GetConfig() {
|
|||||||
}
|
}
|
||||||
Config.OutputPath = *outputPath
|
Config.OutputPath = *outputPath
|
||||||
Config.BatchFilePath = *batchFilePath
|
Config.BatchFilePath = *batchFilePath
|
||||||
|
Config.ScrapedDataFilePath = *scrapedDataFilePath
|
||||||
Config.ArchiveFilePath = *archive
|
Config.ArchiveFilePath = *archive
|
||||||
|
Config.FailLogFilePath = *failLogPath
|
||||||
Config.Debug = *debug
|
Config.Debug = *debug
|
||||||
Config.MetaData = *metadata
|
Config.MetaData = *metadata
|
||||||
Config.Quiet = *quiet
|
Config.Quiet = *quiet
|
||||||
@ -58,3 +66,19 @@ func GetConfig() {
|
|||||||
Config.Deadline = *deadline
|
Config.Deadline = *deadline
|
||||||
Config.Limit = *limit
|
Config.Limit = *limit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func parseBool(str string) bool {
|
||||||
|
val, err := strconv.ParseBool(str)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseInt(str string) int {
|
||||||
|
val, err := strconv.Atoi(str)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return val
|
||||||
|
}
|
||||||
|
@ -5,9 +5,9 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
checkErr "../utils/checkErr"
|
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Upload - Upload object
|
// Upload - Upload object
|
||||||
|
@ -4,8 +4,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
testUtil "../unitTestUtil"
|
testUtil "github.com/pikami/tiktok-dl/unitTestUtil"
|
||||||
fileio "../utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseUploads(t *testing.T) {
|
func TestParseUploads(t *testing.T) {
|
||||||
|
15
package.json
15
package.json
@ -1,14 +1,17 @@
|
|||||||
{
|
{
|
||||||
"name": "tiktok-dl",
|
"name": "tiktok-dl",
|
||||||
"version": "0.0.1",
|
"version": "2.0.0",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"install-dependencies": "go get -v -t -d ./...",
|
"install-dependencies": "go get -v -t -d ./...",
|
||||||
|
"test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
|
||||||
"test": "go test -v ./models && go test -v ./utils",
|
"test": "go test -v ./models && go test -v ./utils",
|
||||||
"clean": "rm -rf out",
|
"clean": "rm -rf out && rm -f scraper.min.js",
|
||||||
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
|
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > scraper.min.js",
|
||||||
"build:app": "go build -o out/ -v .",
|
"build:res": "cd generator && go run . && cd ..",
|
||||||
"build:dist": "mkdir out && npm run build:app && npm run build:scraper",
|
"build:app": "go build -o out/tiktok-dl -v .",
|
||||||
"build": "go build -v ."
|
"build:dist": "npm run clean && mkdir out && npm run build:scraper && npm run build:res && npm run build:app",
|
||||||
|
"build": "go build -v .",
|
||||||
|
"sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"terser": "^4.6.3"
|
"terser": "^4.6.3"
|
||||||
|
22
resources/errorStrings.go
Normal file
22
resources/errorStrings.go
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//ErrorCouldNotSerializeJSON -
|
||||||
|
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
||||||
|
|
||||||
|
//ErrorCouldNotRecogniseURL -
|
||||||
|
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
||||||
|
|
||||||
|
//Error -
|
||||||
|
var Error = "Error : %s\n"
|
||||||
|
|
||||||
|
//ErrorPathNotFound -
|
||||||
|
var ErrorPathNotFound = "File path %s not found."
|
||||||
|
|
||||||
|
//FailedOnItem -
|
||||||
|
var FailedOnItem = "Failed while scraping item: %s\n"
|
||||||
|
|
||||||
|
//FailedToLoadScraper -
|
||||||
|
var FailedToLoadScraper = "Failed to load scraper"
|
103
resources/flags.go
Normal file
103
resources/flags.go
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//FailLogDefault -
|
||||||
|
var FailLogDefault = ""
|
||||||
|
|
||||||
|
//QuietDescription -
|
||||||
|
var QuietDescription = "Suppress output"
|
||||||
|
|
||||||
|
//ScrapedDataDefault -
|
||||||
|
var ScrapedDataDefault = ""
|
||||||
|
|
||||||
|
//ArchiveDescription -
|
||||||
|
var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it."
|
||||||
|
|
||||||
|
//FailLogFlag -
|
||||||
|
var FailLogFlag = "fail-log"
|
||||||
|
|
||||||
|
//QuietDefault -
|
||||||
|
var QuietDefault = "false"
|
||||||
|
|
||||||
|
//JsonFlag -
|
||||||
|
var JsonFlag = "json"
|
||||||
|
|
||||||
|
//ScrapedDataFlag -
|
||||||
|
var ScrapedDataFlag = "scraped-data"
|
||||||
|
|
||||||
|
//DebugFlag -
|
||||||
|
var DebugFlag = "debug"
|
||||||
|
|
||||||
|
//MetadataDefault -
|
||||||
|
var MetadataDefault = "false"
|
||||||
|
|
||||||
|
//DeadlineFlag -
|
||||||
|
var DeadlineFlag = "deadline"
|
||||||
|
|
||||||
|
//OutputFlag -
|
||||||
|
var OutputFlag = "output"
|
||||||
|
|
||||||
|
//BatchFlag -
|
||||||
|
var BatchFlag = "batch-file"
|
||||||
|
|
||||||
|
//ArchiveFlag -
|
||||||
|
var ArchiveFlag = "archive"
|
||||||
|
|
||||||
|
//ArchiveDefault -
|
||||||
|
var ArchiveDefault = ""
|
||||||
|
|
||||||
|
//JsonDefault -
|
||||||
|
var JsonDefault = "false"
|
||||||
|
|
||||||
|
//JsonDescription -
|
||||||
|
var JsonDescription = "Just get JSON data from scraper (without video downloading)"
|
||||||
|
|
||||||
|
//DeadlineDescription -
|
||||||
|
var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
|
||||||
|
|
||||||
|
//OutputDefault -
|
||||||
|
var OutputDefault = "./downloads"
|
||||||
|
|
||||||
|
//OutputDescription -
|
||||||
|
var OutputDescription = "Output path"
|
||||||
|
|
||||||
|
//BatchDescription -
|
||||||
|
var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored."
|
||||||
|
|
||||||
|
//LimitFlag -
|
||||||
|
var LimitFlag = "limit"
|
||||||
|
|
||||||
|
//LimitDescription -
|
||||||
|
var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
|
||||||
|
|
||||||
|
//ScrapedDataDescription -
|
||||||
|
var ScrapedDataDescription = "Download videos from scrape file (json format)"
|
||||||
|
|
||||||
|
//DebugDescription -
|
||||||
|
var DebugDescription = "Enables debug mode"
|
||||||
|
|
||||||
|
//MetadataFlag -
|
||||||
|
var MetadataFlag = "metadata"
|
||||||
|
|
||||||
|
//BatchDefault -
|
||||||
|
var BatchDefault = ""
|
||||||
|
|
||||||
|
//MetadataDescription -
|
||||||
|
var MetadataDescription = "Write video metadata to a .json file"
|
||||||
|
|
||||||
|
//LimitDefault -
|
||||||
|
var LimitDefault = "0"
|
||||||
|
|
||||||
|
//DeadlineDefault -
|
||||||
|
var DeadlineDefault = "1500"
|
||||||
|
|
||||||
|
//FailLogDescription -
|
||||||
|
var FailLogDescription = "Write failed items to log file"
|
||||||
|
|
||||||
|
//DebugDefault -
|
||||||
|
var DebugDefault = "false"
|
||||||
|
|
||||||
|
//QuietFlag -
|
||||||
|
var QuietFlag = "quiet"
|
22
resources/messages.go
Normal file
22
resources/messages.go
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//PreloadingItemsFound -
|
||||||
|
var PreloadingItemsFound = "\rPreloading... %s items have been found."
|
||||||
|
|
||||||
|
//Preloading -
|
||||||
|
var Preloading = "\rPreloading..."
|
||||||
|
|
||||||
|
//Retrieving -
|
||||||
|
var Retrieving = "\nRetrieving items..."
|
||||||
|
|
||||||
|
//ItemsFoundInArchive -
|
||||||
|
var ItemsFoundInArchive = "%d items, found in archive. Skipping...\n"
|
||||||
|
|
||||||
|
//Downloaded -
|
||||||
|
var Downloaded = "\r[%d/%d] Downloaded"
|
||||||
|
|
||||||
|
//UsageLine -
|
||||||
|
var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n or: tiktok-dl [OPTIONS] -scraped-data path/to/data.json"
|
10
resources/scraper.go
Normal file
10
resources/scraper.go
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
// Package resources - This file is automatically generated.
|
||||||
|
// Do not edit this file manually.
|
||||||
|
// Check `/generator/resources.go` to change generated content
|
||||||
|
package resources
|
||||||
|
|
||||||
|
//ScraperPath -
|
||||||
|
var ScraperPath = "scraper.js"
|
||||||
|
|
||||||
|
//ScraperScript -
|
||||||
|
var ScraperScript = "optStrings={selectors:{feedLoading:\".tiktok-ui-loading-container\",modalArrowRight:\"div > div.video-card-container > img.arrow-right\",modalClose:\"div > div.video-card-container > img.control-icon.close\",modalPlayer:\"div.video-card-container > div.video-card-browse > video\",modalCaption:\"div.content-container > div.video-infos-container > h1\",modalSoundLink:\"div.content-container > div.video-infos-container > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > video\",videoCaption:\"div.content-container > div.video-infos-container > h1\",videoSoundLink:\"div.content-container > div.video-infos-container > h2 > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var n=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];!n||n.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):n.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),n=document.querySelector(optStrings.selectors.modalUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),n=document.querySelector(optStrings.selectors.videoUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var n=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(n!==t.count)currentState.preloadCount=t.count;else{if(isLoading())return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},isLoading=function(){var e=document.querySelector(optStrings.selectors.feedLoading);return e&&0!=e.getClientRects().length},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"
|
@ -1,13 +0,0 @@
|
|||||||
package resources
|
|
||||||
|
|
||||||
// ErrorCouldNotSerializeJSON -
|
|
||||||
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
|
||||||
|
|
||||||
// ErrorCouldNotRecogniseURL -
|
|
||||||
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
|
||||||
|
|
||||||
// ErrorCouldNotGetUserUploads -
|
|
||||||
var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
|
|
||||||
|
|
||||||
// ErrorPathNotFound -
|
|
||||||
var ErrorPathNotFound = "File path %s not found."
|
|
79
scraper.js
79
scraper.js
@ -1,17 +1,15 @@
|
|||||||
optStrings = {
|
optStrings = {
|
||||||
selectors: {
|
selectors: {
|
||||||
feedLoading: 'div.tiktok-loading.feed-loading',
|
feedLoading: '.tiktok-ui-loading-container',
|
||||||
modalArrowRight: 'div.video-card-modal > div > img.arrow-right',
|
modalArrowRight: 'div > div.video-card-container > img.arrow-right',
|
||||||
modalClose: '.video-card-modal > div > div.close',
|
modalClose: 'div > div.video-card-container > img.control-icon.close',
|
||||||
modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video',
|
modalPlayer: 'div.video-card-container > div.video-card-browse > video',
|
||||||
modalShareInput: '.copy-link-container > input',
|
modalCaption: 'div.content-container > div.video-infos-container > h1',
|
||||||
modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1',
|
modalSoundLink: 'div.content-container > div.video-infos-container > h2.music-info > a',
|
||||||
modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a',
|
|
||||||
modalUploader: '.user-username',
|
modalUploader: '.user-username',
|
||||||
videoPlayer: 'div.video-card-container > div > div > video',
|
videoPlayer: 'div.video-card-container > div > video',
|
||||||
videoShareInput: 'div.content-container.border > div.copy-link-container > input',
|
videoCaption: 'div.content-container > div.video-infos-container > h1',
|
||||||
videoCaption: 'div.content-container.border > div.video-meta-info > h1',
|
videoSoundLink: 'div.content-container > div.video-infos-container > h2 > a',
|
||||||
videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a',
|
|
||||||
videoUploader: '.user-username',
|
videoUploader: '.user-username',
|
||||||
},
|
},
|
||||||
classes: {
|
classes: {
|
||||||
@ -39,9 +37,9 @@ currentState = {
|
|||||||
limit: 0
|
limit: 0
|
||||||
};
|
};
|
||||||
|
|
||||||
checkForErrors = function() {
|
checkForErrors = function () {
|
||||||
var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
|
var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
|
||||||
debugger;
|
//debugger;
|
||||||
if (titles && titles.length) {
|
if (titles && titles.length) {
|
||||||
var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
|
var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
|
||||||
if (error) {
|
if (error) {
|
||||||
@ -52,21 +50,21 @@ checkForErrors = function() {
|
|||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
createVidUrlElement = function(outputObj) {
|
createVidUrlElement = function (outputObj) {
|
||||||
var urlSetElement = document.createElement(optStrings.tags.resultTag);
|
var urlSetElement = document.createElement(optStrings.tags.resultTag);
|
||||||
urlSetElement.innerText = JSON.stringify(outputObj);
|
urlSetElement.innerText = JSON.stringify(outputObj);
|
||||||
document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
|
document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
|
||||||
currentState.finished = true;
|
currentState.finished = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
buldVidUrlArray = function(finishCallback) {
|
buldVidUrlArray = function (finishCallback) {
|
||||||
var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
|
var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
|
||||||
feedItem.click();
|
feedItem.click();
|
||||||
|
|
||||||
var videoArray = [];
|
var videoArray = [];
|
||||||
var intervalID = window.setInterval(x => {
|
var intervalID = window.setInterval(x => {
|
||||||
videoArray.push(getCurrentModalVideo());
|
videoArray.push(getCurrentModalVideo());
|
||||||
if(currentState.limit > 0) {
|
if (currentState.limit > 0) {
|
||||||
if (videoArray.length >= currentState.limit) {
|
if (videoArray.length >= currentState.limit) {
|
||||||
window.clearInterval(intervalID);
|
window.clearInterval(intervalID);
|
||||||
document.querySelector(optStrings.selectors.modalClose).click();
|
document.querySelector(optStrings.selectors.modalClose).click();
|
||||||
@ -74,7 +72,7 @@ buldVidUrlArray = function(finishCallback) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];
|
var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];
|
||||||
if (arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
|
if (!arrowRight || arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
|
||||||
window.clearInterval(intervalID);
|
window.clearInterval(intervalID);
|
||||||
document.querySelector(optStrings.selectors.modalClose).click();
|
document.querySelector(optStrings.selectors.modalClose).click();
|
||||||
finishCallback(videoArray);
|
finishCallback(videoArray);
|
||||||
@ -84,15 +82,15 @@ buldVidUrlArray = function(finishCallback) {
|
|||||||
}, 20);
|
}, 20);
|
||||||
};
|
};
|
||||||
|
|
||||||
getCurrentModalVideo = function() {
|
getCurrentModalVideo = function () {
|
||||||
var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
|
var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
|
||||||
var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
|
var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
|
||||||
var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value;
|
var shareLink = window.location.href;
|
||||||
var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
|
var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
|
||||||
var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
|
var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
|
||||||
var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent;
|
var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent;
|
||||||
var soundHref = soundLink.getAttribute("href");
|
var soundHref = soundLink ? soundLink.getAttribute("href") : '';
|
||||||
var soundText = soundLink.text;
|
var soundText = soundLink ? soundLink.text : '';
|
||||||
|
|
||||||
return {
|
return {
|
||||||
url: vidUrl,
|
url: vidUrl,
|
||||||
@ -106,16 +104,17 @@ getCurrentModalVideo = function() {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
getCurrentVideo = function() {
|
getCurrentVideo = function () {
|
||||||
if(checkForErrors()) return;
|
//debugger;
|
||||||
|
if (checkForErrors()) return;
|
||||||
var player = document.querySelector(optStrings.selectors.videoPlayer);
|
var player = document.querySelector(optStrings.selectors.videoPlayer);
|
||||||
var vidUrl = player.getAttribute(optStrings.attributes.src);
|
var vidUrl = player.getAttribute(optStrings.attributes.src);
|
||||||
var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
|
var shareLink = window.location.href;
|
||||||
var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
|
var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
|
||||||
var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
|
var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
|
||||||
var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent;
|
var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent;
|
||||||
var soundHref = soundLink.getAttribute("href");
|
var soundHref = soundLink ? soundLink.getAttribute("href") : '';
|
||||||
var soundText = soundLink.text;
|
var soundText = soundLink ? soundLink.text : '';
|
||||||
|
|
||||||
return {
|
return {
|
||||||
url: vidUrl,
|
url: vidUrl,
|
||||||
@ -129,27 +128,32 @@ getCurrentVideo = function() {
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
scrollWhileNew = function(finishCallback) {
|
scrollBottom = () => window.scrollTo(0, document.body.scrollHeight);
|
||||||
var state = { count: 0 };
|
|
||||||
|
scrollWhileNew = function (finishCallback) {
|
||||||
|
var state = {
|
||||||
|
count: 0
|
||||||
|
};
|
||||||
var intervalID = window.setInterval(x => {
|
var intervalID = window.setInterval(x => {
|
||||||
|
scrollBottom();
|
||||||
var oldCount = state.count;
|
var oldCount = state.count;
|
||||||
state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
|
state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
|
||||||
if(currentState.limit > 0) {
|
if (currentState.limit > 0) {
|
||||||
if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) {
|
if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) {
|
||||||
finishCallback(createVidUrlElement);
|
finishCallback(createVidUrlElement);
|
||||||
window.clearInterval(intervalID);
|
window.clearInterval(intervalID);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(checkForErrors()) {
|
if (checkForErrors()) {
|
||||||
window.clearInterval(intervalID);
|
window.clearInterval(intervalID);
|
||||||
return;
|
return;
|
||||||
|
} else if (state.count == 0) {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
if (oldCount !== state.count) {
|
if (oldCount !== state.count) {
|
||||||
currentState.preloadCount = state.count;
|
currentState.preloadCount = state.count;
|
||||||
window.scrollTo(0, document.body.scrollHeight);
|
|
||||||
} else {
|
} else {
|
||||||
if (document.querySelector(optStrings.selectors.feedLoading)) {
|
if (isLoading()) {
|
||||||
window.scrollTo(0, document.body.scrollHeight);
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
window.clearInterval(intervalID);
|
window.clearInterval(intervalID);
|
||||||
@ -158,13 +162,18 @@ scrollWhileNew = function(finishCallback) {
|
|||||||
}, 1000);
|
}, 1000);
|
||||||
};
|
};
|
||||||
|
|
||||||
bootstrapIteratingVideos = function(limit) {
|
isLoading = function () {
|
||||||
|
var loadingElement = document.querySelector(optStrings.selectors.feedLoading);
|
||||||
|
return loadingElement && loadingElement.getClientRects().length != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bootstrapIteratingVideos = function (limit) {
|
||||||
currentState.limit = limit;
|
currentState.limit = limit;
|
||||||
scrollWhileNew(buldVidUrlArray);
|
scrollWhileNew(buldVidUrlArray);
|
||||||
return 'bootstrapIteratingVideos';
|
return 'bootstrapIteratingVideos';
|
||||||
};
|
};
|
||||||
|
|
||||||
bootstrapGetCurrentVideo = function() {
|
bootstrapGetCurrentVideo = function () {
|
||||||
var video = getCurrentVideo();
|
var video = getCurrentVideo();
|
||||||
createVidUrlElement(video);
|
createVidUrlElement(video);
|
||||||
return 'bootstrapGetCurrentVideo';
|
return 'bootstrapGetCurrentVideo';
|
||||||
|
@ -1,10 +1,11 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
models "../models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
fileio "./fileio"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
log "./log"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// IsItemInArchive - Checks if the item is already archived
|
// IsItemInArchive - Checks if the item is already archived
|
||||||
@ -36,7 +37,7 @@ func RemoveArchivedItems(uploads []models.Upload) []models.Upload {
|
|||||||
|
|
||||||
removedCount := lenBeforeRemoval - len(uploads)
|
removedCount := lenBeforeRemoval - len(uploads)
|
||||||
if removedCount > 0 {
|
if removedCount > 0 {
|
||||||
log.Logf("%d items, found in archive. Skipping...\n", removedCount)
|
log.Logf(res.ItemsFoundInArchive, removedCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
return uploads
|
return uploads
|
||||||
|
@ -5,7 +5,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
checkErr "./checkErr"
|
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DownloadFile - Downloads content from `url` and stores it in `outputPath`
|
// DownloadFile - Downloads content from `url` and stores it in `outputPath`
|
||||||
|
@ -5,7 +5,7 @@ import (
|
|||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
checkErr "../checkErr"
|
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||||
)
|
)
|
||||||
|
|
||||||
type delegateString func(string)
|
type delegateString func(string)
|
||||||
|
@ -1,9 +1,10 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
res "../resources"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetHashtagFromURL - Get's tag name from passed url
|
// GetHashtagFromURL - Get's tag name from passed url
|
||||||
|
19
utils/getScraper.go
Normal file
19
utils/getScraper.go
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
resources "github.com/pikami/tiktok-dl/resources"
|
||||||
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetScraper - Retrieve scraper
|
||||||
|
func GetScraper() string {
|
||||||
|
if fileio.CheckIfExists(resources.ScraperPath) {
|
||||||
|
return ReadFileAsString(resources.ScraperPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resources.ScraperScript != "" {
|
||||||
|
return resources.ScraperScript
|
||||||
|
}
|
||||||
|
|
||||||
|
panic(resources.FailedToLoadScraper)
|
||||||
|
}
|
@ -1,11 +1,12 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
config "../models/config"
|
|
||||||
res "../resources"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetUsername - Get's username from passed URL param
|
// GetUsername - Get's username from passed URL param
|
||||||
|
@ -1,9 +1,10 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
config "../models/config"
|
|
||||||
testUtil "../unitTestUtil"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
testUtil "github.com/pikami/tiktok-dl/unitTestUtil"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGetUsername(t *testing.T) {
|
func TestGetUsername(t *testing.T) {
|
||||||
|
@ -4,7 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
config "../../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Log - Write to std out
|
// Log - Write to std out
|
||||||
|
@ -3,7 +3,7 @@ package utils
|
|||||||
import (
|
import (
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
|
||||||
checkErr "./checkErr"
|
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ReadFileAsString - Returns contents of given file
|
// ReadFileAsString - Returns contents of given file
|
||||||
|
20
workflows/common.go
Normal file
20
workflows/common.go
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
package workflows
|
||||||
|
|
||||||
|
import (
|
||||||
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OnWorkflowFail - Function called when workflow fails
|
||||||
|
func OnWorkflowFail(err error, workItem string) {
|
||||||
|
failLogFilePath := config.Config.FailLogFilePath
|
||||||
|
|
||||||
|
if failLogFilePath != "" {
|
||||||
|
fileio.AppendToFile(workItem, failLogFilePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.LogErr(res.Error, err.Error())
|
||||||
|
log.LogErr(res.FailedOnItem, workItem)
|
||||||
|
}
|
@ -1,9 +1,9 @@
|
|||||||
package workflows
|
package workflows
|
||||||
|
|
||||||
import (
|
import (
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
fileio "../utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used
|
// CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used
|
||||||
|
@ -4,12 +4,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
client "../client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "../utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "../utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanUseDownloadHashtag - Test's if this workflow can be used for parameter
|
// CanUseDownloadHashtag - Test's if this workflow can be used for parameter
|
||||||
@ -22,7 +22,7 @@ func CanUseDownloadHashtag(url string) bool {
|
|||||||
func DownloadHashtag(url string) {
|
func DownloadHashtag(url string) {
|
||||||
uploads, err := client.GetHashtagUploads(url)
|
uploads, err := client.GetHashtagUploads(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, url)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -36,7 +36,7 @@ func DownloadHashtag(url string) {
|
|||||||
|
|
||||||
for index, upload := range uploads {
|
for index, upload := range uploads {
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
}
|
}
|
||||||
log.Log()
|
log.Log()
|
||||||
}
|
}
|
||||||
@ -45,7 +45,7 @@ func DownloadHashtag(url string) {
|
|||||||
func GetHashtagJSON(url string) {
|
func GetHashtagJSON(url string) {
|
||||||
uploads, err := client.GetHashtagUploadsJSON(url)
|
uploads, err := client.GetHashtagUploadsJSON(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, url)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Printf("%s", uploads)
|
fmt.Printf("%s", uploads)
|
||||||
|
@ -4,12 +4,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
|
||||||
client "../client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "../utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "../utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
|
// CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
|
||||||
@ -22,7 +22,7 @@ func CanUseDownloadMusic(url string) bool {
|
|||||||
func DownloadMusic(url string) {
|
func DownloadMusic(url string) {
|
||||||
uploads, err := client.GetMusicUploads(url)
|
uploads, err := client.GetMusicUploads(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, url)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -35,7 +35,7 @@ func DownloadMusic(url string) {
|
|||||||
|
|
||||||
fileio.InitOutputDirectory(downloadDir)
|
fileio.InitOutputDirectory(downloadDir)
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
}
|
}
|
||||||
log.Log()
|
log.Log()
|
||||||
}
|
}
|
||||||
@ -44,7 +44,7 @@ func DownloadMusic(url string) {
|
|||||||
func GetMusicJSON(url string) {
|
func GetMusicJSON(url string) {
|
||||||
uploads, err := client.GetMusicUploadsJSON(url)
|
uploads, err := client.GetMusicUploadsJSON(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, url)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Printf("%s", uploads)
|
fmt.Printf("%s", uploads)
|
||||||
|
39
workflows/downloadScrapedData.go
Normal file
39
workflows/downloadScrapedData.go
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
package workflows
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CanUseDownloadScrapedData - Check's if DownloadScrapedData can be used
|
||||||
|
func CanUseDownloadScrapedData(scrapedDataFilePath string) bool {
|
||||||
|
return scrapedDataFilePath != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadScrapedData - Download items from scraped data file
|
||||||
|
func DownloadScrapedData(scrapedDataFilePath string) {
|
||||||
|
if !fileio.CheckIfExists(scrapedDataFilePath) {
|
||||||
|
log.LogFatal(res.ErrorPathNotFound, scrapedDataFilePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
dataFileContent := fileio.ReadFileToString(scrapedDataFilePath)
|
||||||
|
uploads := models.ParseUploads(dataFileContent)
|
||||||
|
uploads = utils.RemoveArchivedItems(uploads)
|
||||||
|
|
||||||
|
uploadCount := len(uploads)
|
||||||
|
|
||||||
|
for index, upload := range uploads {
|
||||||
|
username := utils.GetUsernameFromString(upload.Uploader)
|
||||||
|
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
|
||||||
|
|
||||||
|
fileio.InitOutputDirectory(downloadDir)
|
||||||
|
downloadVideo(upload, downloadDir)
|
||||||
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
|
}
|
||||||
|
log.Log()
|
||||||
|
}
|
@ -1,10 +1,10 @@
|
|||||||
package workflows
|
package workflows
|
||||||
|
|
||||||
import (
|
import (
|
||||||
client "../client"
|
|
||||||
res "../resources"
|
|
||||||
log "../utils/log"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
|
|
||||||
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanUseDownloadShareLink - Check's if DownloadShareLink can be used
|
// CanUseDownloadShareLink - Check's if DownloadShareLink can be used
|
||||||
@ -17,9 +17,9 @@ func CanUseDownloadShareLink(url string) bool {
|
|||||||
func DownloadShareLink(url string) {
|
func DownloadShareLink(url string) {
|
||||||
log.Logf("Resolving share link: %s\n", url)
|
log.Logf("Resolving share link: %s\n", url)
|
||||||
|
|
||||||
finalURL, err := client.GetRedirectUrl(url)
|
finalURL, err := client.GetRedirectURL(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, url)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -5,12 +5,12 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
client "../client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "../utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "../utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanUseDownloadUser - Test's if this workflow can be used for parameter
|
// CanUseDownloadUser - Test's if this workflow can be used for parameter
|
||||||
@ -24,7 +24,7 @@ func CanUseDownloadUser(url string) bool {
|
|||||||
func DownloadUser(username string) {
|
func DownloadUser(username string) {
|
||||||
uploads, err := client.GetUserUploads(username)
|
uploads, err := client.GetUserUploads(username)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, username)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -37,7 +37,7 @@ func DownloadUser(username string) {
|
|||||||
|
|
||||||
for index, upload := range uploads {
|
for index, upload := range uploads {
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
|
log.Logf(res.Downloaded, index+1, uploadCount)
|
||||||
}
|
}
|
||||||
log.Log()
|
log.Log()
|
||||||
}
|
}
|
||||||
@ -46,7 +46,7 @@ func DownloadUser(username string) {
|
|||||||
func GetUserVideosJSON(username string) {
|
func GetUserVideosJSON(username string) {
|
||||||
uploads, err := client.GetUserUploadsJSON(username)
|
uploads, err := client.GetUserUploadsJSON(username)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, username)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Printf("%s", uploads)
|
fmt.Printf("%s", uploads)
|
||||||
|
@ -4,13 +4,13 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
|
||||||
client "../client"
|
client "github.com/pikami/tiktok-dl/client"
|
||||||
models "../models"
|
models "github.com/pikami/tiktok-dl/models"
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "../utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
fileio "../utils/fileio"
|
fileio "github.com/pikami/tiktok-dl/utils/fileio"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
|
// CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
|
||||||
@ -24,7 +24,7 @@ func DownloadSingleVideo(url string) {
|
|||||||
username := utils.GetUsernameFromString(url)
|
username := utils.GetUsernameFromString(url)
|
||||||
upload, err := client.GetVideoDetails(url)
|
upload, err := client.GetVideoDetails(url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
OnWorkflowFail(err, url)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -35,7 +35,7 @@ func DownloadSingleVideo(url string) {
|
|||||||
|
|
||||||
fileio.InitOutputDirectory(downloadDir)
|
fileio.InitOutputDirectory(downloadDir)
|
||||||
downloadVideo(upload, downloadDir)
|
downloadVideo(upload, downloadDir)
|
||||||
log.Log("[1/1] Downloaded\n")
|
log.Logf(res.Downloaded, 1, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadVideo - Downloads one video
|
// DownloadVideo - Downloads one video
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
package workflows
|
package workflows
|
||||||
|
|
||||||
import (
|
import (
|
||||||
config "../models/config"
|
config "github.com/pikami/tiktok-dl/models/config"
|
||||||
res "../resources"
|
res "github.com/pikami/tiktok-dl/resources"
|
||||||
utils "../utils"
|
utils "github.com/pikami/tiktok-dl/utils"
|
||||||
log "../utils/log"
|
log "github.com/pikami/tiktok-dl/utils/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// StartWorkflowByParameter - Start needed workflow by given parameter
|
// StartWorkflowByParameter - Start needed workflow by given parameter
|
||||||
|
Loading…
x
Reference in New Issue
Block a user