From 39cfeb40fb59226c107f991a095ed413e27c8c23 Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Sun, 19 Jan 2020 17:54:16 +0200 Subject: [PATCH] Added ability to download single videos --- .gitignore | 4 ++- README.md | 17 +++++++++++ client/getUserUploads.go | 5 ++-- client/getVideoDetails.go | 58 ++++++++++++++++++++++++++++++++++++ main.go | 48 ++++++++++++++++++++++------- models/config.go | 6 ++-- models/upload.go | 7 +++++ getVidLinks.js => scraper.js | 25 ++++++++++++++-- 8 files changed, 150 insertions(+), 20 deletions(-) create mode 100644 README.md create mode 100644 client/getVideoDetails.go rename getVidLinks.js => scraper.js (77%) diff --git a/.gitignore b/.gitignore index 446b7fe..93c0d8b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.vscode +__debug_bin downloads *.exe -tiktok-dl \ No newline at end of file +tiktok-dl diff --git a/README.md b/README.md new file mode 100644 index 0000000..a81eee0 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# TikTok-DL + +[![Go Report Card](https://goreportcard.com/badge/github.com/pikami/tiktok-dl)](https://goreportcard.com/report/github.com/pikami/tiktok-dl) + +A simple tiktok video downloader written in go + +## Basic usage +Clone this repository and run `go build` to build the executable.\ +You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\ +You can download single video by running `./tiktok-dl [Options] VIDEO_URL` + +## Available options +* `-debug` - enables debug mode +* `-output some_directory` - Output path (default "./downloads") + +## Acknowledgments +This software uses the chromedp for web scraping, it can be found here: https://github.com/chromedp/chromedp \ No newline at end of file diff --git a/client/getUserUploads.go b/client/getUserUploads.go index bdf67a1..7e37f09 100644 --- a/client/getUserUploads.go +++ b/client/getUserUploads.go @@ -23,7 +23,7 @@ func GetUserUploads(username string) []models.Upload { opts := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.DisableGPU, chromedp.UserDataDir(dir), - chromedp.Flag("headless", models.Config.UserName), + chromedp.Flag("headless", !models.Config.Debug), ) allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) @@ -43,7 +43,8 @@ func GetUserUploads(username string) []models.Upload { // Navigate to user's page chromedp.Navigate(`https://www.tiktok.com/@`+username), // Execute url grabber script - chromedp.EvaluateAsDevTools(utils.ReadFileAsString("getVidLinks.js"), &jsOutput), + chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), + chromedp.EvaluateAsDevTools("bootstrapIteratingVideos()", &jsOutput), // Wait until custom js finishes chromedp.WaitVisible(`video_urls`), // Grab url links from our element diff --git a/client/getVideoDetails.go b/client/getVideoDetails.go new file mode 100644 index 0000000..1bb5af1 --- /dev/null +++ b/client/getVideoDetails.go @@ -0,0 +1,58 @@ +package client + +import ( + "context" + "github.com/chromedp/chromedp" + "io/ioutil" + "log" + "os" + "time" + + models "../models" + utils "../utils" +) + +// GetVideoDetails - returns details of video +func GetVideoDetails(videoURL string) models.Upload { + dir, err := ioutil.TempDir("", "chromedp-example") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.DisableGPU, + chromedp.UserDataDir(dir), + chromedp.Flag("headless", !models.Config.Debug), + ) + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancel() + + ctx, cancel := chromedp.NewContext( + allocCtx, + chromedp.WithLogf(log.Printf), + ) + defer cancel() + + ctx, cancel = context.WithTimeout(ctx, 1500*time.Second) + defer cancel() + + var jsOutput string + err = chromedp.Run(ctx, + // Navigate to user's page + chromedp.Navigate(videoURL), + // Execute url grabber script + chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), + chromedp.EvaluateAsDevTools("bootstrapGetCurrentVideo()", &jsOutput), + // Wait until custom js finishes + chromedp.WaitVisible(`video_urls`), + // Grab url links from our element + chromedp.InnerHTML(`video_urls`, &jsOutput), + ) + if err != nil { + log.Fatal(err) + } + + return models.ParseUpload(jsOutput) +} diff --git a/main.go b/main.go index d5332ac..ba48c5e 100644 --- a/main.go +++ b/main.go @@ -5,27 +5,53 @@ import ( models "./models" utils "./utils" "fmt" + "regexp" + "strings" ) func main() { models.GetConfig() + url := models.Config.URL - username := models.Config.UserName + // Single video + match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url) + if match { + getUsernameFromVidURLRegex, _ := regexp.Compile("com\\/@.*") + parts := strings.Split(getUsernameFromVidURLRegex.FindString(url), "/") + username := parts[1][1:] + upload := client.GetVideoDetails(url) + downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username) + + utils.InitOutputDirectory(downloadDir) + downloadVideo(upload, downloadDir) + return + } + + // Tiktok user + downloadUser() +} + +func downloadVideo(upload models.Upload, downloadDir string) { + uploadID := upload.GetUploadID() + downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID) + + if utils.CheckIfExists(downloadPath) { + fmt.Println("Upload '" + uploadID + "' already downloaded, skipping") + return + } + + fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath) + utils.DownloadFile(downloadPath, upload.URL) +} + +func downloadUser() { + username := models.Config.URL downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username) uploads := client.GetUserUploads(username) utils.InitOutputDirectory(downloadDir) for _, upload := range uploads { - uploadID := upload.GetUploadID() - downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID) - - if utils.CheckIfExists(downloadPath) { - fmt.Println("Upload '" + uploadID + "' already downloaded, skipping") - continue - } - - fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath) - utils.DownloadFile(downloadPath, upload.URL) + downloadVideo(upload, downloadDir) } } diff --git a/models/config.go b/models/config.go index e3c6075..ad7b5e6 100644 --- a/models/config.go +++ b/models/config.go @@ -8,7 +8,7 @@ import ( // Config - Runtime configuration var Config struct { - UserName string + URL string OutputPath string Debug bool } @@ -21,11 +21,11 @@ func GetConfig() { args := flag.Args() if len(args) < 1 { - fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME") + fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL") os.Exit(2) } - Config.UserName = flag.Args()[len(args)-1] + Config.URL = flag.Args()[len(args)-1] Config.OutputPath = *outputPath Config.Debug = *debug } diff --git a/models/upload.go b/models/upload.go index 315ab92..3b90241 100644 --- a/models/upload.go +++ b/models/upload.go @@ -18,6 +18,13 @@ func ParseUploads(str string) []Upload { return uploads } +// ParseUpload - Parses json uploads array +func ParseUpload(str string) Upload { + var upload Upload + json.Unmarshal([]byte(str), &upload) + return upload +} + // GetUploadID - Returns upload id func (u Upload) GetUploadID() string { parts := strings.Split(u.ShareLink, "/") diff --git a/getVidLinks.js b/scraper.js similarity index 77% rename from getVidLinks.js rename to scraper.js index 9924131..6e13a31 100644 --- a/getVidLinks.js +++ b/scraper.js @@ -5,6 +5,8 @@ optStrings = { modalClose: '.video-card-modal > div > div.close', modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video', modalShareInput: '.copy-link-container > input', + videoPlayer: 'div.video-card-container > div > div > video', + videoShareInput: 'div.content-container.border > div.copy-link-container > input', }, classes: { modalCloseDisabled: 'disabled', @@ -18,9 +20,9 @@ optStrings = { }, }; -createVidUrlElement = function(videoArray) { +createVidUrlElement = function(outputObj) { var urlSetElement = document.createElement(optStrings.tags.resultTag); - urlSetElement.innerText = JSON.stringify(videoArray); + urlSetElement.innerText = JSON.stringify(outputObj); document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement); } @@ -54,6 +56,17 @@ getCurrentModalVideo = function() { }; } +getCurrentVideo = function() { + var player = document.querySelector(optStrings.selectors.videoPlayer); + var vidUrl = player.getAttribute(optStrings.attributes.src); + var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value; + + return { + url: vidUrl, + shareLink: shareLink + }; +} + scrollWhileNew = function(finishCallback) { var state = { count: 0 }; var intervalID = window.setInterval(x => { @@ -76,13 +89,19 @@ bootstrapIteratingVideos = function() { window.clearInterval(intervalID); } }, 500); + return 'bootstrapIteratingVideos'; }; +bootstrapGetCurrentVideo = function() { + var video = getCurrentVideo(); + createVidUrlElement(video); + return 'bootstrapGetCurrentVideo'; +} + init = () => { const newProto = navigator.__proto__; delete newProto.webdriver; navigator.__proto__ = newProto; - bootstrapIteratingVideos(); }; init();