pk
/
tiktok-dl
mirror of https://github.com/pikami/tiktok-dl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

formatter

This commit is contained in:
alexpin 2020-02-25 01:01:10 +02:00
parent f724f0f2a2
commit b6bb470064
10 changed files with 212 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
client/executeClientAction.go
View File

@ -1,102 +1,102 @@
package client
import (
"context"
"github.com/chromedp/chromedp"
"io/ioutil"
"log"
"os"
"time"
"context"
"github.com/chromedp/chromedp"
"io/ioutil"
"log"
"os"
"time"
config "../models/config"
utils "../utils"
config "../models/config"
utils "../utils"
)
// GetMusicUploads - Get all uploads by given music
func executeClientAction(url string, jsAction string) string {
dir, err := ioutil.TempDir("", "chromedp-example")
utils.CheckErr(err)
defer os.RemoveAll(dir)
dir, err := ioutil.TempDir("", "chromedp-example")
utils.CheckErr(err)
defer os.RemoveAll(dir)
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.DisableGPU,
chromedp.UserDataDir(dir),
chromedp.Flag("headless", !config.Config.Debug),
)
opts := append(chromedp.DefaultExecAllocatorOptions[:],
chromedp.DisableGPU,
chromedp.UserDataDir(dir),
chromedp.Flag("headless", !config.Config.Debug),
)
allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancel()
allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...)
defer cancel()
ctx, cancel := chromedp.NewContext(
allocCtx,
chromedp.WithLogf(log.Printf),
)
defer cancel()
ctx, cancel := chromedp.NewContext(
allocCtx,
chromedp.WithLogf(log.Printf),
)
defer cancel()
ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
defer cancel()
ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
defer cancel()
var jsOutput string
jsOutput = runScrapeWithInfo(ctx, jsAction, url)
var jsOutput string
jsOutput = runScrapeWithInfo(ctx, jsAction, url)
return jsOutput
return jsOutput
}
func runScrapeQuiet(ctx context.Context, jsAction string, url string) string {
var jsOutput string
err := chromedp.Run(ctx,
// Navigate to user's page
chromedp.Navigate(url),
// Execute url grabber script
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
// Wait until custom js finishes
chromedp.WaitVisible(`video_urls`),
// Grab url links from our element
chromedp.InnerHTML(`video_urls`, &jsOutput),
)
utils.CheckErr(err)
return jsOutput
var jsOutput string
err := chromedp.Run(ctx,
// Navigate to user's page
chromedp.Navigate(url),
// Execute url grabber script
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
// Wait until custom js finishes
chromedp.WaitVisible(`video_urls`),
// Grab url links from our element
chromedp.InnerHTML(`video_urls`, &jsOutput),
)
utils.CheckErr(err)
return jsOutput
}
func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string {
var jsOutput string
err := chromedp.Run(ctx,
// Navigate to user's page
chromedp.Navigate(url),
// Execute url grabber script
chromedp.WaitReady("video"),
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
)
utils.CheckErr(err)
var jsOutput string
err := chromedp.Run(ctx,
// Navigate to user's page
chromedp.Navigate(url),
// Execute url grabber script
chromedp.WaitReady("video"),
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
)
utils.CheckErr(err)
for {
err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput))
utils.CheckErr(err)
if jsOutput != "0" {
utils.Logf("\rPreloading... %s items have been founded.", jsOutput)
} else {
utils.Logf("\rPreloading...")
}
for {
err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput))
utils.CheckErr(err)
if jsOutput != "0" {
utils.Logf("\rPreloading... %s items have been founded.", jsOutput)
} else {
utils.Logf("\rPreloading...")
}
err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput))
utils.CheckErr(err)
if jsOutput == "true" {
break
}
err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput))
utils.CheckErr(err)
if jsOutput == "true" {
break
}
time.Sleep(50 * time.Millisecond)
}
time.Sleep(50 * time.Millisecond)
}
utils.Log("\nRetrieving items...")
err = chromedp.Run(ctx,
// Wait until custom js finishes
chromedp.WaitVisible(`video_urls`),
// Grab url links from our element
chromedp.InnerHTML(`video_urls`, &jsOutput),
)
utils.CheckErr(err)
utils.Log("\nRetrieving items...")
err = chromedp.Run(ctx,
// Wait until custom js finishes
chromedp.WaitVisible(`video_urls`),
// Grab url links from our element
chromedp.InnerHTML(`video_urls`, &jsOutput),
)
utils.CheckErr(err)
return jsOutput
return jsOutput
}

18
client/getHashtagUploads.go
View File

@ -1,19 +1,19 @@
package client
import (
models "../models"
config "../models/config"
"fmt"
models "../models"
config "../models/config"
"fmt"
)
// GetUserUploads - Get all uploads marked with given hashtag
func GetHashtagUploads(hashtagURL string) []models.Upload {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput := executeClientAction(hashtagURL, jsMethod)
return models.ParseUploads(actionOutput)
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput := executeClientAction(hashtagURL, jsMethod)
return models.ParseUploads(actionOutput)
}
func GetHashtagUploadsJson(hashtagURL string) string {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
return executeClientAction(hashtagURL, jsMethod)
}
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
return executeClientAction(hashtagURL, jsMethod)
}

18
client/getMusicUploads.go
View File

@ -1,19 +1,19 @@
package client
import (
models "../models"
config "../models/config"
"fmt"
models "../models"
config "../models/config"
"fmt"
)
// GetMusicUploads - Get all uploads by given music
func GetMusicUploads(url string) []models.Upload {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput := executeClientAction(url, jsMethod)
return models.ParseUploads(actionOutput)
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput := executeClientAction(url, jsMethod)
return models.ParseUploads(actionOutput)
}
func GetMusicUploadsJson(url string) string {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
return executeClientAction(url, jsMethod)
}
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
return executeClientAction(url, jsMethod)
}

8
client/getUserUploads.go
View File

@ -3,17 +3,17 @@ package client
import (
models "../models"
config "../models/config"
"fmt"
"fmt"
)
// GetUserUploads - Get all uploads by user
func GetUserUploads(username string) []models.Upload {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
return models.ParseUploads(actionOutput)
}
func GetUserUploadsJson(username string) string {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
return executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
}
}

86
models/config/config.go
View File

@ -1,57 +1,57 @@
package config
import (
"flag"
"fmt"
"os"
"flag"
"fmt"
"os"
)
// Config - Runtime configuration
var Config struct {
URL string
OutputPath string
BatchFilePath string
Debug bool
MetaData bool
Quiet bool
Deadline int
Limit int
JSONOnly bool
URL string
OutputPath string
BatchFilePath string
Debug bool
MetaData bool
Quiet bool
Deadline int
Limit int
JSONOnly bool
}
// GetConfig - Returns Config object
func GetConfig() {
outputPath := flag.String("output", "./downloads", "Output path")
batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
debug := flag.Bool("debug", false, "Enables debug mode")
metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
quiet := flag.Bool("quiet", false, "Supress output")
deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
flag.Parse()
outputPath := flag.String("output", "./downloads", "Output path")
batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.")
debug := flag.Bool("debug", false, "Enables debug mode")
metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
quiet := flag.Bool("quiet", false, "Supress output")
deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)")
limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)")
jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)")
flag.Parse()
args := flag.Args()
if len(args) < 1 && *batchFilePath == "" {
fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
os.Exit(2)
}
args := flag.Args()
if len(args) < 1 && *batchFilePath == "" {
fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL")
fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt")
os.Exit(2)
}
if len(args) > 0 {
Config.URL = flag.Args()[len(args)-1]
} else {
Config.URL = ""
}
Config.OutputPath = *outputPath
Config.BatchFilePath = *batchFilePath
Config.Debug = *debug
Config.MetaData = *metadata
Config.Quiet = *quiet
if *jsonOnly {
Config.Quiet = true
}
Config.Deadline = *deadline
Config.Limit = *limit
Config.JSONOnly = *jsonOnly;
if len(args) > 0 {
Config.URL = flag.Args()[len(args)-1]
} else {
Config.URL = ""
}
Config.OutputPath = *outputPath
Config.BatchFilePath = *batchFilePath
Config.Debug = *debug
Config.MetaData = *metadata
Config.Quiet = *quiet
if *jsonOnly {
Config.Quiet = true
}
Config.Deadline = *deadline
Config.Limit = *limit
Config.JSONOnly = *jsonOnly
}

14
utils/getHashtag.go
View File

@ -1,16 +1,16 @@
package utils
import (
res "../resources"
"fmt"
"strings"
res "../resources"
"fmt"
"strings"
)
// GetHashtagFromURL - Get's tag name from passed url
func GetHashtagFromURL(str string) string {
if match := strings.Contains(str, "/tag/"); match {
return strings.Split(str, "/tag/")[1]
}
if match := strings.Contains(str, "/tag/"); match {
return strings.Split(str, "/tag/")[1]
}
panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str))
panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str))
}

38
workflows/downloadHashtag.go
View File

@ -1,36 +1,36 @@
package workflows
import (
client "../client"
config "../models/config"
utils "../utils"
"fmt"
"strings"
client "../client"
config "../models/config"
utils "../utils"
"fmt"
"strings"
)
// CanUseDownloadHashtag - Test's if this workflow can be used for parameter
func CanUseDownloadHashtag(url string) bool {
match := strings.Contains(url, "/tag/")
return match
match := strings.Contains(url, "/tag/")
return match
}
// DownloadHashtag - Download videos marked with given hashtag
func DownloadHashtag(url string) {
uploads := client.GetHashtagUploads(url)
uploadCount := len(uploads)
hashtag := utils.GetHashtagFromURL(url)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, hashtag)
uploads := client.GetHashtagUploads(url)
uploadCount := len(uploads)
hashtag := utils.GetHashtagFromURL(url)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, hashtag)
utils.InitOutputDirectory(downloadDir)
utils.InitOutputDirectory(downloadDir)
for index, upload := range uploads {
downloadVideo(upload, downloadDir)
utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
}
utils.Log()
for index, upload := range uploads {
downloadVideo(upload, downloadDir)
utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
}
utils.Log()
}
func GetHashtagJson(url string) {
uploads := client.GetHashtagUploads(url)
fmt.Printf("%s", uploads)
uploads := client.GetHashtagUploads(url)
fmt.Printf("%s", uploads)
}

38
workflows/downloadMusic.go
View File

@ -1,36 +1,36 @@
package workflows
import (
client "../client"
config "../models/config"
utils "../utils"
"fmt"
"regexp"
client "../client"
config "../models/config"
utils "../utils"
"fmt"
"regexp"
)
// CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter
func CanUseDownloadMusic(url string) bool {
match, _ := regexp.MatchString(".com\\/music\\/.+", url)
return match
match, _ := regexp.MatchString(".com\\/music\\/.+", url)
return match
}
// DownloadMusic - Download all videos by given music
func DownloadMusic(url string) {
uploads := client.GetMusicUploads(url)
uploadCount := len(uploads)
uploads := client.GetMusicUploads(url)
uploadCount := len(uploads)
for index, upload := range uploads {
username := utils.GetUsernameFromString(upload.Uploader)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
for index, upload := range uploads {
username := utils.GetUsernameFromString(upload.Uploader)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
utils.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir)
utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
}
utils.Log()
utils.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir)
utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount)
}
utils.Log()
}
func GetMusicJson(url string) {
uploads := client.GetMusicUploadsJson(url)
fmt.Printf("%s", uploads)
uploads := client.GetMusicUploadsJson(url)
fmt.Printf("%s", uploads)
}

2
workflows/downloadUser.go
View File

@ -31,5 +31,5 @@ func DownloadUser(username string) {
func GetUserVideosJson(username string) {
uploads := client.GetUserUploadsJson(username)
fmt.Printf("%s", uploads)
fmt.Printf("%s", uploads)
}

50
workflows/downloadVideo.go
View File

@ -1,44 +1,44 @@
package workflows
import (
client "../client"
models "../models"
config "../models/config"
utils "../utils"
"fmt"
"regexp"
client "../client"
models "../models"
config "../models/config"
utils "../utils"
"fmt"
"regexp"
)
// CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter
func CanUseDownloadSingleVideo(url string) bool {
match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url)
return match
match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url)
return match
}
// DownloadSingleVideo - Downloads single video
func DownloadSingleVideo(url string) {
username := utils.GetUsernameFromString(url)
upload := client.GetVideoDetails(url)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
username := utils.GetUsernameFromString(url)
upload := client.GetVideoDetails(url)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
utils.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir)
utils.Log("[1/1] Downloaded\n")
utils.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir)
utils.Log("[1/1] Downloaded\n")
}
// DownloadVideo - Downloads one video
func downloadVideo(upload models.Upload, downloadDir string) {
uploadID := upload.GetUploadID()
downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
uploadID := upload.GetUploadID()
downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID)
if utils.CheckIfExists(downloadPath) {
return
}
if utils.CheckIfExists(downloadPath) {
return
}
utils.DownloadFile(downloadPath, upload.URL)
utils.DownloadFile(downloadPath, upload.URL)
if config.Config.MetaData {
metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
upload.WriteToFile(metadataPath)
}
}
if config.Config.MetaData {
metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
upload.WriteToFile(metadataPath)
}
}