Compare commits

...

11 Commits
1.9 ... master

Author SHA1 Message Date
Pijus Kamandulis
e2e4ba0d4b Added ability to download from scrapeData (json) 2020-08-16 18:56:30 +03:00
Pijus Kamandulis
7a25f521cb Updated scraper.js 2020-08-16 01:30:48 +03:00
Pijus Kamandulis
e9715c0d40
Update README.md 2020-05-02 09:48:13 +03:00
Pijus Kamandulis
a70dc4cf04
Update README.md 2020-05-02 09:46:43 +03:00
Pijus Kamandulis
ea2b866f9a Embed scraper into compiled binary, extract strings to resources file 2020-04-12 16:56:50 +03:00
Pijus Kamandulis
70c3faf17e Use go modules 2020-04-12 03:22:00 +03:00
Pijus Kamandulis
0a08f62dfe Fix lint issues 2020-04-09 18:53:45 +03:00
Pijus Kamandulis
bb6e924866
Add execute permission to binary file durring build 2020-04-09 18:32:11 +03:00
Pijus Kamandulis
af59659f41 TTDL-17 Added option to output failed items to file 2020-04-09 18:10:33 +03:00
Pijus Kamandulis
92006d864f TTDL-20 Wait for error/items beefore quiting 2020-04-08 23:26:50 +03:00
Pijus Kamandulis
668b050dee Added sonar-scaner
Create LICENSE

Update go.yml
2020-04-04 21:09:41 +03:00
44 changed files with 679 additions and 175 deletions

View File

@ -34,6 +34,12 @@ jobs:
- name: Build - name: Build
run: npm run build:dist run: npm run build:dist
- name: Copy license
run: cp LICENSE out
- name: Add execute permission
run: chmod +x out/tiktok-dl
- name: Upload Unix Artifacts - name: Upload Unix Artifacts
if: startsWith(matrix.os, 'ubuntu-') if: startsWith(matrix.os, 'ubuntu-')
uses: actions/upload-artifact@v1 uses: actions/upload-artifact@v1

2
.gitignore vendored
View File

@ -6,3 +6,5 @@ downloads
tiktok-dl tiktok-dl
batch_file.txt batch_file.txt
debug.log debug.log
.scannerwork
scraper.min.js

View File

@ -0,0 +1,12 @@
sonar.organization=pikami
sonar.projectKey=tiktok-dl
sonar.host.url=https://sonarcloud.io
sonar.sources=.
sonar.exclusions=**/*_test.go,**/node_modules/**
sonar.tests=.
sonar.test.inclusions=**/*_test.go
sonar.test.exclusions=**/node_modules/**
sonar.go.coverage.reportPaths=cov.out

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 pikami
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -1,30 +1,44 @@
# TikTok-DL # TikTok-DL
[![Go Report Card](https://goreportcard.com/badge/github.com/pikami/tiktok-dl)](https://goreportcard.com/report/github.com/pikami/tiktok-dl) [![Go Report Card](https://goreportcard.com/badge/github.com/pikami/tiktok-dl)](https://goreportcard.com/report/github.com/pikami/tiktok-dl)
![GitHub Workflow Status](https://img.shields.io/github/workflow/status/pikami/tiktok-dl/tiktok-dl_CI) [![tiktok-dl_CI](https://github.com/pikami/tiktok-dl/workflows/tiktok-dl_CI/badge.svg?branch=master)](https://github.com/pikami/tiktok-dl/actions)
A simple tiktok video downloader written in go A simple tiktok video downloader written in go
## Basic usage ```diff
Download the executable from `https://github.com/pikami/tiktok-dl/releases`\ - This tool is not working currenly, I will revive it in the future when I have some free time
You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\ ```
You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\
You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\
You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt`
## Build instructions ## Basic usage examples
Clone this repository and run `go build` to build the executable. Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\
You can download single video by running `./tiktok-dl VIDEO_URL`\
You can download items listed in a text file by running `./tiktok-dl -batch-file path/to/items.txt`
## Usage Manual
```
Usage: tiktok-dl [OPTION]... TARGET
or: tiktok-dl [OPTION]... -batch-file BATCH_FILE
In the 1st form, download given `TARGET`.
In the 2nd form, download all targets listed in given `BATCH_FILE`.
```
## Available options ## Available options
* `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it. * `-archive` - Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
* `-batch-file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored. * `-batch-file some_file` - File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.
* `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500) * `-deadline` - Sets the timout for scraper logic in seconds (used as a workaround for context deadline exceeded error) (default 1500)
* `-debug` - enables debug mode * `-debug` - enables debug mode
* `-fail-log some_file` - Write failed items to log file
* `-json` - Returns whole data, that was scraped from TikTok, in json * `-json` - Returns whole data, that was scraped from TikTok, in json
* `-limit` - Sets the max count of video that will be downloaded (default infinity) * `-limit` - Sets the max count of video that will be downloaded (default infinity)
* `-metadata` - Write video metadata to a .json file * `-metadata` - Write video metadata to a .json file
* `-output some_directory` - Output path (default "./downloads") * `-output some_directory` - Output path (default "./downloads")
* `-quiet` - Supress output * `-quiet` - Suppress output
## Build instructions
1. Clone this repository
2. Run `go build` to build the executable.
## Acknowledgments ## Acknowledgments
This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \ This software uses the **chromedp** for web scraping, it can be found here: https://github.com/chromedp/chromedp \

View File

@ -10,9 +10,10 @@ import (
"github.com/chromedp/chromedp" "github.com/chromedp/chromedp"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
utils "../utils" res "github.com/pikami/tiktok-dl/resources"
log "../utils/log" utils "github.com/pikami/tiktok-dl/utils"
log "github.com/pikami/tiktok-dl/utils/log"
) )
// GetMusicUploads - Get all uploads by given music // GetMusicUploads - Get all uploads by given music
@ -54,7 +55,7 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, e
// Navigate to user's page // Navigate to user's page
chromedp.Navigate(url), chromedp.Navigate(url),
// Execute url grabber script // Execute url grabber script
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
chromedp.EvaluateAsDevTools(jsAction, &jsOutput), chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
// Wait until custom js finishes // Wait until custom js finishes
chromedp.WaitVisible(`video_urls`), chromedp.WaitVisible(`video_urls`),
@ -73,7 +74,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
// Navigate to user's page // Navigate to user's page
chromedp.Navigate(url), chromedp.Navigate(url),
// Execute url grabber script // Execute url grabber script
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), chromedp.EvaluateAsDevTools(utils.GetScraper(), &jsOutput),
chromedp.EvaluateAsDevTools(jsAction, &jsOutput), chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
); err != nil { ); err != nil {
return "", err return "", err
@ -85,9 +86,9 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
} }
if jsOutput != "0" { if jsOutput != "0" {
log.Logf("\rPreloading... %s items have been found.", jsOutput) log.Logf(res.PreloadingItemsFound, jsOutput)
} else { } else {
log.Logf("\rPreloading...") log.Logf(res.Preloading)
} }
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil { if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
@ -101,7 +102,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
time.Sleep(50 * time.Millisecond) time.Sleep(50 * time.Millisecond)
} }
log.Log("\nRetrieving items...") log.Log(res.Retrieving)
if err := chromedp.Run(ctx, if err := chromedp.Run(ctx,
// Wait until custom js finishes // Wait until custom js finishes
chromedp.WaitVisible(`video_urls`), chromedp.WaitVisible(`video_urls`),

View File

@ -3,8 +3,8 @@ package client
import ( import (
"fmt" "fmt"
models "../models" models "github.com/pikami/tiktok-dl/models"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
) )
// GetHashtagUploads - Get all uploads marked with given hashtag // GetHashtagUploads - Get all uploads marked with given hashtag

View File

@ -3,8 +3,8 @@ package client
import ( import (
"fmt" "fmt"
models "../models" models "github.com/pikami/tiktok-dl/models"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
) )
// GetMusicUploads - Get all uploads by given music // GetMusicUploads - Get all uploads by given music

View File

@ -2,16 +2,18 @@ package client
import ( import (
"context" "context"
"github.com/chromedp/chromedp"
"io/ioutil" "io/ioutil"
"os" "os"
"time" "time"
config "../models/config" "github.com/chromedp/chromedp"
log "../utils/log"
config "github.com/pikami/tiktok-dl/models/config"
log "github.com/pikami/tiktok-dl/utils/log"
) )
func GetRedirectUrl(url string) (string, error) { // GetRedirectURL - Returns URL that the given URL redirects to
func GetRedirectURL(url string) (string, error) {
dir, err := ioutil.TempDir("", "chromedp-example") dir, err := ioutil.TempDir("", "chromedp-example")
if err != nil { if err != nil {
return "", err return "", err

View File

@ -3,8 +3,8 @@ package client
import ( import (
"fmt" "fmt"
models "../models" models "github.com/pikami/tiktok-dl/models"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
) )
// GetUserUploads - Get all uploads by user // GetUserUploads - Get all uploads by user

View File

@ -1,7 +1,7 @@
package client package client
import ( import (
models "../models" models "github.com/pikami/tiktok-dl/models"
) )
// GetVideoDetails - returns details of video // GetVideoDetails - returns details of video

64
generator/generator.go Normal file
View File

@ -0,0 +1,64 @@
package main
import (
"fmt"
"os"
"strings"
checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
fileio "github.com/pikami/tiktok-dl/utils/fileio"
)
type resource struct {
Package string
FileName string
Values map[string]string
}
func (r resource) generate() {
filename := fmt.Sprintf("%s/%s", outputDir, r.FileName)
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
checkErr.CheckErr(err)
defer f.Close()
// Header
header := fmt.Sprintf("// Package %s - This file is automatically generated.\n"+
"// Do not edit this file manually.\n"+
"// Check `/generator/resources.go` to change generated content\n"+
"package %s\n", r.Package, r.Package)
if _, err := f.WriteString(header); err != nil {
checkErr.CheckErr(err)
}
// Values
for key, value := range r.Values {
value = strings.ReplaceAll(value, "\n", "\\n")
value = strings.ReplaceAll(value, "\r", "\\r")
valueLine := fmt.Sprintf("\n//%s -\nvar %s = \"%s\"\n", key, key, value)
if _, err := f.WriteString(valueLine); err != nil {
checkErr.CheckErr(err)
}
}
}
func fileContentsOrDefault(file string) string {
defer func() {
if r := recover(); r != nil {
fmt.Printf("Failed to load file: %s\n", file)
}
}()
return safeString(fileio.ReadFileToString(file))
}
func safeString(str string) string {
escaped := strings.ReplaceAll(str, "\"", "\\\"")
return strings.ReplaceAll(escaped, "\n", "")
}
func main() {
for _, i := range res {
i.generate()
}
}

92
generator/resources.go Normal file
View File

@ -0,0 +1,92 @@
package main
var (
outputDir = "../resources"
res = []resource{
resource{
Package: "resources",
FileName: "scraper.go",
Values: map[string]string{
"ScraperPath": "scraper.js",
"ScraperScript": fileContentsOrDefault("../scraper.min.js"),
},
},
resource{
Package: "resources",
FileName: "errorStrings.go",
Values: map[string]string{
"ErrorCouldNotSerializeJSON": "Could not serialize json for video: %s\n",
"ErrorCouldNotRecogniseURL": "Could not recognise URL format of string %s",
"Error": "Error : %s\n",
"ErrorPathNotFound": "File path %s not found.",
"FailedOnItem": "Failed while scraping item: %s\n",
"FailedToLoadScraper": "Failed to load scraper",
},
},
resource{
Package: "resources",
FileName: "messages.go",
Values: map[string]string{
"PreloadingItemsFound": "\rPreloading... %s items have been found.",
"Preloading": "\rPreloading...",
"Retrieving": "\nRetrieving items...",
"ItemsFoundInArchive": "%d items, found in archive. Skipping...\n",
"Downloaded": "\r[%d/%d] Downloaded",
"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n" +
" or: tiktok-dl [OPTIONS] -scraped-data path/to/data.json",
},
},
resource{
Package: "resources",
FileName: "flags.go",
Values: map[string]string{
// Output
"OutputFlag": "output",
"OutputDefault": "./downloads",
"OutputDescription": "Output path",
// Batch file
"BatchFlag": "batch-file",
"BatchDefault": "",
"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
// ScrapedData
"ScrapedDataFlag": "scraped-data",
"ScrapedDataDefault": "",
"ScrapedDataDescription": "Download videos from scrape file (json format)",
// Archive
"ArchiveFlag": "archive",
"ArchiveDefault": "",
"ArchiveDescription": "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.",
// Fail log
"FailLogFlag": "fail-log",
"FailLogDefault": "",
"FailLogDescription": "Write failed items to log file",
// Debug
"DebugFlag": "debug",
"DebugDefault": "false",
"DebugDescription": "Enables debug mode",
// Metadata
"MetadataFlag": "metadata",
"MetadataDefault": "false",
"MetadataDescription": "Write video metadata to a .json file",
// Quiet
"QuietFlag": "quiet",
"QuietDefault": "false",
"QuietDescription": "Suppress output",
// JSON only
"JsonFlag": "json",
"JsonDefault": "false",
"JsonDescription": "Just get JSON data from scraper (without video downloading)",
// Deadline
"DeadlineFlag": "deadline",
"DeadlineDefault": "1500",
"DeadlineDescription": "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)",
// Limit
"LimitFlag": "limit",
"LimitDefault": "0",
"LimitDescription": "Sets the videos count limit (useful when there too many videos from the user or by hashtag)",
},
},
}
)

5
go.mod Normal file
View File

@ -0,0 +1,5 @@
module github.com/pikami/tiktok-dl
go 1.14
require github.com/chromedp/chromedp v0.5.3

16
go.sum Normal file
View File

@ -0,0 +1,16 @@
github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac h1:T7V5BXqnYd55Hj/g5uhDYumg9Fp3rMTS6bykYtTIFX4=
github.com/chromedp/cdproto v0.0.0-20200116234248-4da64dd111ac/go.mod h1:PfAWWKJqjlGFYJEidUM6aVIWPr0EpobeyVWEEmplX7g=
github.com/chromedp/chromedp v0.5.3 h1:F9LafxmYpsQhWQBdCs+6Sret1zzeeFyHS5LkRF//Ffg=
github.com/chromedp/chromedp v0.5.3/go.mod h1:YLdPtndaHQ4rCpSpBG+IPpy9JvX0VD+7aaLxYgYj28w=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 h1:V0an7KRw92wmJysvFvtqtKMAPmvS5O0jtB0nYo6t+gs=
github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

11
main.go
View File

@ -1,14 +1,15 @@
package main package main
import ( import (
config "./models/config" config "github.com/pikami/tiktok-dl/models/config"
workflows "./workflows" workflows "github.com/pikami/tiktok-dl/workflows"
) )
func main() { func main() {
config.GetConfig() config.GetConfig()
url := config.Config.URL url := config.Config.URL
batchFilePath := config.Config.BatchFilePath batchFilePath := config.Config.BatchFilePath
scrapedDataFilePath := config.Config.ScrapedDataFilePath
// Batch file // Batch file
if workflows.CanUseDownloadBatchFile(batchFilePath) { if workflows.CanUseDownloadBatchFile(batchFilePath) {
@ -16,5 +17,11 @@ func main() {
return return
} }
// Scraped data file
if workflows.CanUseDownloadScrapedData(scrapedDataFilePath) {
workflows.DownloadScrapedData(scrapedDataFilePath)
return
}
workflows.StartWorkflowByParameter(url) workflows.StartWorkflowByParameter(url)
} }

View File

@ -4,6 +4,9 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strconv"
res "github.com/pikami/tiktok-dl/resources"
) )
// Config - Runtime configuration // Config - Runtime configuration
@ -11,7 +14,9 @@ var Config struct {
URL string URL string
OutputPath string OutputPath string
BatchFilePath string BatchFilePath string
ScrapedDataFilePath string
ArchiveFilePath string ArchiveFilePath string
FailLogFilePath string
Debug bool Debug bool
MetaData bool MetaData bool
Quiet bool Quiet bool
@ -22,21 +27,22 @@ var Config struct {
// GetConfig - Returns Config object // GetConfig - Returns Config object
func GetConfig() { func GetConfig() {
outputPath := flag.String("output", "./downloads", "Output path") outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription)
batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.") batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription)
archive := flag.String("archive", "", "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.") scrapedDataFilePath := flag.String(res.ScrapedDataFlag, res.ScrapedDataDefault, res.ScrapedDataDescription)
debug := flag.Bool("debug", false, "Enables debug mode") archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription)
metadata := flag.Bool("metadata", false, "Write video metadata to a .json file") failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription)
quiet := flag.Bool("quiet", false, "Supress output") debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription)
jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)") metadata := flag.Bool(res.MetadataFlag, parseBool(res.MetadataDefault), res.MetadataDescription)
deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)") quiet := flag.Bool(res.QuietFlag, parseBool(res.QuietDefault), res.QuietDescription)
limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)") jsonOnly := flag.Bool(res.JsonFlag, parseBool(res.JsonDefault), res.JsonDescription)
deadline := flag.Int(res.DeadlineFlag, parseInt(res.DeadlineDefault), res.DeadlineDescription)
limit := flag.Int(res.LimitFlag, parseInt(res.LimitDefault), res.LimitDescription)
flag.Parse() flag.Parse()
args := flag.Args() args := flag.Args()
if len(args) < 1 && *batchFilePath == "" { if len(args) < 1 && *batchFilePath == "" && *scrapedDataFilePath == "" {
fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL") fmt.Println(res.UsageLine)
fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
os.Exit(2) os.Exit(2)
} }
@ -47,7 +53,9 @@ func GetConfig() {
} }
Config.OutputPath = *outputPath Config.OutputPath = *outputPath
Config.BatchFilePath = *batchFilePath Config.BatchFilePath = *batchFilePath
Config.ScrapedDataFilePath = *scrapedDataFilePath
Config.ArchiveFilePath = *archive Config.ArchiveFilePath = *archive
Config.FailLogFilePath = *failLogPath
Config.Debug = *debug Config.Debug = *debug
Config.MetaData = *metadata Config.MetaData = *metadata
Config.Quiet = *quiet Config.Quiet = *quiet
@ -58,3 +66,19 @@ func GetConfig() {
Config.Deadline = *deadline Config.Deadline = *deadline
Config.Limit = *limit Config.Limit = *limit
} }
func parseBool(str string) bool {
val, err := strconv.ParseBool(str)
if err != nil {
panic(err)
}
return val
}
func parseInt(str string) int {
val, err := strconv.Atoi(str)
if err != nil {
panic(err)
}
return val
}

View File

@ -5,9 +5,9 @@ import (
"os" "os"
"strings" "strings"
res "../resources" res "github.com/pikami/tiktok-dl/resources"
checkErr "../utils/checkErr" checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// Upload - Upload object // Upload - Upload object

View File

@ -4,8 +4,8 @@ import (
"os" "os"
"testing" "testing"
testUtil "../unitTestUtil" testUtil "github.com/pikami/tiktok-dl/unitTestUtil"
fileio "../utils/fileio" fileio "github.com/pikami/tiktok-dl/utils/fileio"
) )
func TestParseUploads(t *testing.T) { func TestParseUploads(t *testing.T) {

View File

@ -1,14 +1,17 @@
{ {
"name": "tiktok-dl", "name": "tiktok-dl",
"version": "0.0.1", "version": "2.0.0",
"scripts": { "scripts": {
"install-dependencies": "go get -v -t -d ./...", "install-dependencies": "go get -v -t -d ./...",
"test:coverage": "go test -short -coverprofile=cov.out ./models ./utils",
"test": "go test -v ./models && go test -v ./utils", "test": "go test -v ./models && go test -v ./utils",
"clean": "rm -rf out", "clean": "rm -rf out && rm -f scraper.min.js",
"build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js", "build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > scraper.min.js",
"build:app": "go build -o out/ -v .", "build:res": "cd generator && go run . && cd ..",
"build:dist": "mkdir out && npm run build:app && npm run build:scraper", "build:app": "go build -o out/tiktok-dl -v .",
"build": "go build -v ." "build:dist": "npm run clean && mkdir out && npm run build:scraper && npm run build:res && npm run build:app",
"build": "go build -v .",
"sonar": "sonar-scanner -Dsonar.login=${SONAR_LOGIN} -Dproject.settings=.sonar/sonar-project.properties"
}, },
"dependencies": { "dependencies": {
"terser": "^4.6.3" "terser": "^4.6.3"

22
resources/errorStrings.go Normal file
View File

@ -0,0 +1,22 @@
// Package resources - This file is automatically generated.
// Do not edit this file manually.
// Check `/generator/resources.go` to change generated content
package resources
//ErrorCouldNotSerializeJSON -
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
//ErrorCouldNotRecogniseURL -
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
//Error -
var Error = "Error : %s\n"
//ErrorPathNotFound -
var ErrorPathNotFound = "File path %s not found."
//FailedOnItem -
var FailedOnItem = "Failed while scraping item: %s\n"
//FailedToLoadScraper -
var FailedToLoadScraper = "Failed to load scraper"

103
resources/flags.go Normal file
View File

@ -0,0 +1,103 @@
// Package resources - This file is automatically generated.
// Do not edit this file manually.
// Check `/generator/resources.go` to change generated content
package resources
//FailLogDefault -
var FailLogDefault = ""
//QuietDescription -
var QuietDescription = "Suppress output"
//ScrapedDataDefault -
var ScrapedDataDefault = ""
//ArchiveDescription -
var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it."
//FailLogFlag -
var FailLogFlag = "fail-log"
//QuietDefault -
var QuietDefault = "false"
//JsonFlag -
var JsonFlag = "json"
//ScrapedDataFlag -
var ScrapedDataFlag = "scraped-data"
//DebugFlag -
var DebugFlag = "debug"
//MetadataDefault -
var MetadataDefault = "false"
//DeadlineFlag -
var DeadlineFlag = "deadline"
//OutputFlag -
var OutputFlag = "output"
//BatchFlag -
var BatchFlag = "batch-file"
//ArchiveFlag -
var ArchiveFlag = "archive"
//ArchiveDefault -
var ArchiveDefault = ""
//JsonDefault -
var JsonDefault = "false"
//JsonDescription -
var JsonDescription = "Just get JSON data from scraper (without video downloading)"
//DeadlineDescription -
var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
//OutputDefault -
var OutputDefault = "./downloads"
//OutputDescription -
var OutputDescription = "Output path"
//BatchDescription -
var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored."
//LimitFlag -
var LimitFlag = "limit"
//LimitDescription -
var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
//ScrapedDataDescription -
var ScrapedDataDescription = "Download videos from scrape file (json format)"
//DebugDescription -
var DebugDescription = "Enables debug mode"
//MetadataFlag -
var MetadataFlag = "metadata"
//BatchDefault -
var BatchDefault = ""
//MetadataDescription -
var MetadataDescription = "Write video metadata to a .json file"
//LimitDefault -
var LimitDefault = "0"
//DeadlineDefault -
var DeadlineDefault = "1500"
//FailLogDescription -
var FailLogDescription = "Write failed items to log file"
//DebugDefault -
var DebugDefault = "false"
//QuietFlag -
var QuietFlag = "quiet"

22
resources/messages.go Normal file
View File

@ -0,0 +1,22 @@
// Package resources - This file is automatically generated.
// Do not edit this file manually.
// Check `/generator/resources.go` to change generated content
package resources
//PreloadingItemsFound -
var PreloadingItemsFound = "\rPreloading... %s items have been found."
//Preloading -
var Preloading = "\rPreloading..."
//Retrieving -
var Retrieving = "\nRetrieving items..."
//ItemsFoundInArchive -
var ItemsFoundInArchive = "%d items, found in archive. Skipping...\n"
//Downloaded -
var Downloaded = "\r[%d/%d] Downloaded"
//UsageLine -
var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n or: tiktok-dl [OPTIONS] -scraped-data path/to/data.json"

10
resources/scraper.go Normal file
View File

@ -0,0 +1,10 @@
// Package resources - This file is automatically generated.
// Do not edit this file manually.
// Check `/generator/resources.go` to change generated content
package resources
//ScraperPath -
var ScraperPath = "scraper.js"
//ScraperScript -
var ScraperScript = "optStrings={selectors:{feedLoading:\".tiktok-ui-loading-container\",modalArrowRight:\"div > div.video-card-container > img.arrow-right\",modalClose:\"div > div.video-card-container > img.control-icon.close\",modalPlayer:\"div.video-card-container > div.video-card-browse > video\",modalCaption:\"div.content-container > div.video-infos-container > h1\",modalSoundLink:\"div.content-container > div.video-infos-container > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > video\",videoCaption:\"div.content-container > div.video-infos-container > h1\",videoSoundLink:\"div.content-container > div.video-infos-container > h2 > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var n=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];!n||n.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):n.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),n=document.querySelector(optStrings.selectors.modalUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),n=document.querySelector(optStrings.selectors.videoUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var n=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(n!==t.count)currentState.preloadCount=t.count;else{if(isLoading())return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},isLoading=function(){var e=document.querySelector(optStrings.selectors.feedLoading);return e&&0!=e.getClientRects().length},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"

View File

@ -1,13 +0,0 @@
package resources
// ErrorCouldNotSerializeJSON -
var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
// ErrorCouldNotRecogniseURL -
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
// ErrorCouldNotGetUserUploads -
var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
// ErrorPathNotFound -
var ErrorPathNotFound = "File path %s not found."

View File

@ -1,17 +1,15 @@
optStrings = { optStrings = {
selectors: { selectors: {
feedLoading: 'div.tiktok-loading.feed-loading', feedLoading: '.tiktok-ui-loading-container',
modalArrowRight: 'div.video-card-modal > div > img.arrow-right', modalArrowRight: 'div > div.video-card-container > img.arrow-right',
modalClose: '.video-card-modal > div > div.close', modalClose: 'div > div.video-card-container > img.control-icon.close',
modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video', modalPlayer: 'div.video-card-container > div.video-card-browse > video',
modalShareInput: '.copy-link-container > input', modalCaption: 'div.content-container > div.video-infos-container > h1',
modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1', modalSoundLink: 'div.content-container > div.video-infos-container > h2.music-info > a',
modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a',
modalUploader: '.user-username', modalUploader: '.user-username',
videoPlayer: 'div.video-card-container > div > div > video', videoPlayer: 'div.video-card-container > div > video',
videoShareInput: 'div.content-container.border > div.copy-link-container > input', videoCaption: 'div.content-container > div.video-infos-container > h1',
videoCaption: 'div.content-container.border > div.video-meta-info > h1', videoSoundLink: 'div.content-container > div.video-infos-container > h2 > a',
videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a',
videoUploader: '.user-username', videoUploader: '.user-username',
}, },
classes: { classes: {
@ -39,9 +37,9 @@ currentState = {
limit: 0 limit: 0
}; };
checkForErrors = function() { checkForErrors = function () {
var titles = document.getElementsByClassName(optStrings.classes.titleMessage); var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
debugger; //debugger;
if (titles && titles.length) { if (titles && titles.length) {
var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent; var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
if (error) { if (error) {
@ -52,21 +50,21 @@ checkForErrors = function() {
return false; return false;
}; };
createVidUrlElement = function(outputObj) { createVidUrlElement = function (outputObj) {
var urlSetElement = document.createElement(optStrings.tags.resultTag); var urlSetElement = document.createElement(optStrings.tags.resultTag);
urlSetElement.innerText = JSON.stringify(outputObj); urlSetElement.innerText = JSON.stringify(outputObj);
document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement); document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
currentState.finished = true; currentState.finished = true;
}; };
buldVidUrlArray = function(finishCallback) { buldVidUrlArray = function (finishCallback) {
var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0]; var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
feedItem.click(); feedItem.click();
var videoArray = []; var videoArray = [];
var intervalID = window.setInterval(x => { var intervalID = window.setInterval(x => {
videoArray.push(getCurrentModalVideo()); videoArray.push(getCurrentModalVideo());
if(currentState.limit > 0) { if (currentState.limit > 0) {
if (videoArray.length >= currentState.limit) { if (videoArray.length >= currentState.limit) {
window.clearInterval(intervalID); window.clearInterval(intervalID);
document.querySelector(optStrings.selectors.modalClose).click(); document.querySelector(optStrings.selectors.modalClose).click();
@ -74,7 +72,7 @@ buldVidUrlArray = function(finishCallback) {
} }
} }
var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0]; var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];
if (arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) { if (!arrowRight || arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
window.clearInterval(intervalID); window.clearInterval(intervalID);
document.querySelector(optStrings.selectors.modalClose).click(); document.querySelector(optStrings.selectors.modalClose).click();
finishCallback(videoArray); finishCallback(videoArray);
@ -84,15 +82,15 @@ buldVidUrlArray = function(finishCallback) {
}, 20); }, 20);
}; };
getCurrentModalVideo = function() { getCurrentModalVideo = function () {
var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer); var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src); var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value; var shareLink = window.location.href;
var caption = document.querySelector(optStrings.selectors.modalCaption).textContent; var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
var soundLink = document.querySelector(optStrings.selectors.modalSoundLink); var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent; var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent;
var soundHref = soundLink.getAttribute("href"); var soundHref = soundLink ? soundLink.getAttribute("href") : '';
var soundText = soundLink.text; var soundText = soundLink ? soundLink.text : '';
return { return {
url: vidUrl, url: vidUrl,
@ -106,16 +104,17 @@ getCurrentModalVideo = function() {
}; };
}; };
getCurrentVideo = function() { getCurrentVideo = function () {
if(checkForErrors()) return; //debugger;
if (checkForErrors()) return;
var player = document.querySelector(optStrings.selectors.videoPlayer); var player = document.querySelector(optStrings.selectors.videoPlayer);
var vidUrl = player.getAttribute(optStrings.attributes.src); var vidUrl = player.getAttribute(optStrings.attributes.src);
var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value; var shareLink = window.location.href;
var caption = document.querySelector(optStrings.selectors.videoCaption).textContent; var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
var soundLink = document.querySelector(optStrings.selectors.videoSoundLink); var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent; var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent;
var soundHref = soundLink.getAttribute("href"); var soundHref = soundLink ? soundLink.getAttribute("href") : '';
var soundText = soundLink.text; var soundText = soundLink ? soundLink.text : '';
return { return {
url: vidUrl, url: vidUrl,
@ -129,27 +128,32 @@ getCurrentVideo = function() {
}; };
}; };
scrollWhileNew = function(finishCallback) { scrollBottom = () => window.scrollTo(0, document.body.scrollHeight);
var state = { count: 0 };
scrollWhileNew = function (finishCallback) {
var state = {
count: 0
};
var intervalID = window.setInterval(x => { var intervalID = window.setInterval(x => {
scrollBottom();
var oldCount = state.count; var oldCount = state.count;
state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length; state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
if(currentState.limit > 0) { if (currentState.limit > 0) {
if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) { if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) {
finishCallback(createVidUrlElement); finishCallback(createVidUrlElement);
window.clearInterval(intervalID); window.clearInterval(intervalID);
} }
} }
if(checkForErrors()) { if (checkForErrors()) {
window.clearInterval(intervalID); window.clearInterval(intervalID);
return; return;
} else if (state.count == 0) {
return;
} }
if (oldCount !== state.count) { if (oldCount !== state.count) {
currentState.preloadCount = state.count; currentState.preloadCount = state.count;
window.scrollTo(0, document.body.scrollHeight);
} else { } else {
if (document.querySelector(optStrings.selectors.feedLoading)) { if (isLoading()) {
window.scrollTo(0, document.body.scrollHeight);
return; return;
} }
window.clearInterval(intervalID); window.clearInterval(intervalID);
@ -158,13 +162,18 @@ scrollWhileNew = function(finishCallback) {
}, 1000); }, 1000);
}; };
bootstrapIteratingVideos = function(limit) { isLoading = function () {
var loadingElement = document.querySelector(optStrings.selectors.feedLoading);
return loadingElement && loadingElement.getClientRects().length != 0;
}
bootstrapIteratingVideos = function (limit) {
currentState.limit = limit; currentState.limit = limit;
scrollWhileNew(buldVidUrlArray); scrollWhileNew(buldVidUrlArray);
return 'bootstrapIteratingVideos'; return 'bootstrapIteratingVideos';
}; };
bootstrapGetCurrentVideo = function() { bootstrapGetCurrentVideo = function () {
var video = getCurrentVideo(); var video = getCurrentVideo();
createVidUrlElement(video); createVidUrlElement(video);
return 'bootstrapGetCurrentVideo'; return 'bootstrapGetCurrentVideo';

View File

@ -1,10 +1,11 @@
package utils package utils
import ( import (
models "../models" models "github.com/pikami/tiktok-dl/models"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
fileio "./fileio" res "github.com/pikami/tiktok-dl/resources"
log "./log" fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "github.com/pikami/tiktok-dl/utils/log"
) )
// IsItemInArchive - Checks if the item is already archived // IsItemInArchive - Checks if the item is already archived
@ -36,7 +37,7 @@ func RemoveArchivedItems(uploads []models.Upload) []models.Upload {
removedCount := lenBeforeRemoval - len(uploads) removedCount := lenBeforeRemoval - len(uploads)
if removedCount > 0 { if removedCount > 0 {
log.Logf("%d items, found in archive. Skipping...\n", removedCount) log.Logf(res.ItemsFoundInArchive, removedCount)
} }
return uploads return uploads

View File

@ -5,7 +5,7 @@ import (
"net/http" "net/http"
"os" "os"
checkErr "./checkErr" checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
) )
// DownloadFile - Downloads content from `url` and stores it in `outputPath` // DownloadFile - Downloads content from `url` and stores it in `outputPath`

View File

@ -5,7 +5,7 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
checkErr "../checkErr" checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
) )
type delegateString func(string) type delegateString func(string)

View File

@ -1,9 +1,10 @@
package utils package utils
import ( import (
res "../resources"
"fmt" "fmt"
"strings" "strings"
res "github.com/pikami/tiktok-dl/resources"
) )
// GetHashtagFromURL - Get's tag name from passed url // GetHashtagFromURL - Get's tag name from passed url

19
utils/getScraper.go Normal file
View File

@ -0,0 +1,19 @@
package utils
import (
resources "github.com/pikami/tiktok-dl/resources"
fileio "github.com/pikami/tiktok-dl/utils/fileio"
)
// GetScraper - Retrieve scraper
func GetScraper() string {
if fileio.CheckIfExists(resources.ScraperPath) {
return ReadFileAsString(resources.ScraperPath)
}
if resources.ScraperScript != "" {
return resources.ScraperScript
}
panic(resources.FailedToLoadScraper)
}

View File

@ -1,11 +1,12 @@
package utils package utils
import ( import (
config "../models/config"
res "../resources"
"fmt" "fmt"
"regexp" "regexp"
"strings" "strings"
config "github.com/pikami/tiktok-dl/models/config"
res "github.com/pikami/tiktok-dl/resources"
) )
// GetUsername - Get's username from passed URL param // GetUsername - Get's username from passed URL param

View File

@ -1,9 +1,10 @@
package utils package utils
import ( import (
config "../models/config"
testUtil "../unitTestUtil"
"testing" "testing"
config "github.com/pikami/tiktok-dl/models/config"
testUtil "github.com/pikami/tiktok-dl/unitTestUtil"
) )
func TestGetUsername(t *testing.T) { func TestGetUsername(t *testing.T) {

View File

@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"os" "os"
config "../../models/config" config "github.com/pikami/tiktok-dl/models/config"
) )
// Log - Write to std out // Log - Write to std out

View File

@ -3,7 +3,7 @@ package utils
import ( import (
"io/ioutil" "io/ioutil"
checkErr "./checkErr" checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
) )
// ReadFileAsString - Returns contents of given file // ReadFileAsString - Returns contents of given file

20
workflows/common.go Normal file
View File

@ -0,0 +1,20 @@
package workflows
import (
config "github.com/pikami/tiktok-dl/models/config"
res "github.com/pikami/tiktok-dl/resources"
fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "github.com/pikami/tiktok-dl/utils/log"
)
// OnWorkflowFail - Function called when workflow fails
func OnWorkflowFail(err error, workItem string) {
failLogFilePath := config.Config.FailLogFilePath
if failLogFilePath != "" {
fileio.AppendToFile(workItem, failLogFilePath)
}
log.LogErr(res.Error, err.Error())
log.LogErr(res.FailedOnItem, workItem)
}

View File

@ -1,9 +1,9 @@
package workflows package workflows
import ( import (
res "../resources" res "github.com/pikami/tiktok-dl/resources"
fileio "../utils/fileio" fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used // CanUseDownloadBatchFile - Check's if DownloadBatchFile can be used

View File

@ -4,12 +4,12 @@ import (
"fmt" "fmt"
"strings" "strings"
client "../client" client "github.com/pikami/tiktok-dl/client"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
res "../resources" res "github.com/pikami/tiktok-dl/resources"
utils "../utils" utils "github.com/pikami/tiktok-dl/utils"
fileio "../utils/fileio" fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// CanUseDownloadHashtag - Test's if this workflow can be used for parameter // CanUseDownloadHashtag - Test's if this workflow can be used for parameter
@ -22,7 +22,7 @@ func CanUseDownloadHashtag(url string) bool {
func DownloadHashtag(url string) { func DownloadHashtag(url string) {
uploads, err := client.GetHashtagUploads(url) uploads, err := client.GetHashtagUploads(url)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, url)
return return
} }
@ -36,7 +36,7 @@ func DownloadHashtag(url string) {
for index, upload := range uploads { for index, upload := range uploads {
downloadVideo(upload, downloadDir) downloadVideo(upload, downloadDir)
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) log.Logf(res.Downloaded, index+1, uploadCount)
} }
log.Log() log.Log()
} }
@ -45,7 +45,7 @@ func DownloadHashtag(url string) {
func GetHashtagJSON(url string) { func GetHashtagJSON(url string) {
uploads, err := client.GetHashtagUploadsJSON(url) uploads, err := client.GetHashtagUploadsJSON(url)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, url)
return return
} }
fmt.Printf("%s", uploads) fmt.Printf("%s", uploads)

View File

@ -4,12 +4,12 @@ import (
"fmt" "fmt"
"regexp" "regexp"
client "../client" client "github.com/pikami/tiktok-dl/client"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
res "../resources" res "github.com/pikami/tiktok-dl/resources"
utils "../utils" utils "github.com/pikami/tiktok-dl/utils"
fileio "../utils/fileio" fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter // CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
@ -22,7 +22,7 @@ func CanUseDownloadMusic(url string) bool {
func DownloadMusic(url string) { func DownloadMusic(url string) {
uploads, err := client.GetMusicUploads(url) uploads, err := client.GetMusicUploads(url)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, url)
return return
} }
@ -35,7 +35,7 @@ func DownloadMusic(url string) {
fileio.InitOutputDirectory(downloadDir) fileio.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir) downloadVideo(upload, downloadDir)
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) log.Logf(res.Downloaded, index+1, uploadCount)
} }
log.Log() log.Log()
} }
@ -44,7 +44,7 @@ func DownloadMusic(url string) {
func GetMusicJSON(url string) { func GetMusicJSON(url string) {
uploads, err := client.GetMusicUploadsJSON(url) uploads, err := client.GetMusicUploadsJSON(url)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, url)
return return
} }
fmt.Printf("%s", uploads) fmt.Printf("%s", uploads)

View File

@ -0,0 +1,39 @@
package workflows
import (
"fmt"
models "github.com/pikami/tiktok-dl/models"
config "github.com/pikami/tiktok-dl/models/config"
res "github.com/pikami/tiktok-dl/resources"
utils "github.com/pikami/tiktok-dl/utils"
fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "github.com/pikami/tiktok-dl/utils/log"
)
// CanUseDownloadScrapedData - Check's if DownloadScrapedData can be used
func CanUseDownloadScrapedData(scrapedDataFilePath string) bool {
return scrapedDataFilePath != ""
}
// DownloadScrapedData - Download items from scraped data file
func DownloadScrapedData(scrapedDataFilePath string) {
if !fileio.CheckIfExists(scrapedDataFilePath) {
log.LogFatal(res.ErrorPathNotFound, scrapedDataFilePath)
}
dataFileContent := fileio.ReadFileToString(scrapedDataFilePath)
uploads := models.ParseUploads(dataFileContent)
uploads = utils.RemoveArchivedItems(uploads)
uploadCount := len(uploads)
for index, upload := range uploads {
username := utils.GetUsernameFromString(upload.Uploader)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
fileio.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir)
log.Logf(res.Downloaded, index+1, uploadCount)
}
log.Log()
}

View File

@ -1,10 +1,10 @@
package workflows package workflows
import ( import (
client "../client"
res "../resources"
log "../utils/log"
"regexp" "regexp"
client "github.com/pikami/tiktok-dl/client"
log "github.com/pikami/tiktok-dl/utils/log"
) )
// CanUseDownloadShareLink - Check's if DownloadShareLink can be used // CanUseDownloadShareLink - Check's if DownloadShareLink can be used
@ -17,9 +17,9 @@ func CanUseDownloadShareLink(url string) bool {
func DownloadShareLink(url string) { func DownloadShareLink(url string) {
log.Logf("Resolving share link: %s\n", url) log.Logf("Resolving share link: %s\n", url)
finalURL, err := client.GetRedirectUrl(url) finalURL, err := client.GetRedirectURL(url)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, url)
return return
} }

View File

@ -5,12 +5,12 @@ import (
"regexp" "regexp"
"strings" "strings"
client "../client" client "github.com/pikami/tiktok-dl/client"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
res "../resources" res "github.com/pikami/tiktok-dl/resources"
utils "../utils" utils "github.com/pikami/tiktok-dl/utils"
fileio "../utils/fileio" fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// CanUseDownloadUser - Test's if this workflow can be used for parameter // CanUseDownloadUser - Test's if this workflow can be used for parameter
@ -24,7 +24,7 @@ func CanUseDownloadUser(url string) bool {
func DownloadUser(username string) { func DownloadUser(username string) {
uploads, err := client.GetUserUploads(username) uploads, err := client.GetUserUploads(username)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, username)
return return
} }
@ -37,7 +37,7 @@ func DownloadUser(username string) {
for index, upload := range uploads { for index, upload := range uploads {
downloadVideo(upload, downloadDir) downloadVideo(upload, downloadDir)
log.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) log.Logf(res.Downloaded, index+1, uploadCount)
} }
log.Log() log.Log()
} }
@ -46,7 +46,7 @@ func DownloadUser(username string) {
func GetUserVideosJSON(username string) { func GetUserVideosJSON(username string) {
uploads, err := client.GetUserUploadsJSON(username) uploads, err := client.GetUserUploadsJSON(username)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, username)
return return
} }
fmt.Printf("%s", uploads) fmt.Printf("%s", uploads)

View File

@ -4,13 +4,13 @@ import (
"fmt" "fmt"
"regexp" "regexp"
client "../client" client "github.com/pikami/tiktok-dl/client"
models "../models" models "github.com/pikami/tiktok-dl/models"
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
res "../resources" res "github.com/pikami/tiktok-dl/resources"
utils "../utils" utils "github.com/pikami/tiktok-dl/utils"
fileio "../utils/fileio" fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter // CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
@ -24,7 +24,7 @@ func DownloadSingleVideo(url string) {
username := utils.GetUsernameFromString(url) username := utils.GetUsernameFromString(url)
upload, err := client.GetVideoDetails(url) upload, err := client.GetVideoDetails(url)
if err != nil { if err != nil {
log.LogErr(res.ErrorCouldNotGetUserUploads, err.Error()) OnWorkflowFail(err, url)
return return
} }
@ -35,7 +35,7 @@ func DownloadSingleVideo(url string) {
fileio.InitOutputDirectory(downloadDir) fileio.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir) downloadVideo(upload, downloadDir)
log.Log("[1/1] Downloaded\n") log.Logf(res.Downloaded, 1, 1)
} }
// DownloadVideo - Downloads one video // DownloadVideo - Downloads one video

View File

@ -1,10 +1,10 @@
package workflows package workflows
import ( import (
config "../models/config" config "github.com/pikami/tiktok-dl/models/config"
res "../resources" res "github.com/pikami/tiktok-dl/resources"
utils "../utils" utils "github.com/pikami/tiktok-dl/utils"
log "../utils/log" log "github.com/pikami/tiktok-dl/utils/log"
) )
// StartWorkflowByParameter - Start needed workflow by given parameter // StartWorkflowByParameter - Start needed workflow by given parameter