From 69574dcb7ed270f7dfb4b4e4bfffa08075c3a2a2 Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Sun, 19 Jan 2020 04:11:53 +0200 Subject: [PATCH] Initial commit --- .gitignore | 3 ++ client/getUserUploads.go | 57 +++++++++++++++++++++++++++++++++ getVidLinks.js | 67 +++++++++++++++++++++++++++++++++++++++ main.go | 30 ++++++++++++++++++ models/config.go | 28 ++++++++++++++++ models/upload.go | 25 +++++++++++++++ models/upload_test.go | 36 +++++++++++++++++++++ utils/downloadFile.go | 31 ++++++++++++++++++ utils/fileio.go | 21 ++++++++++++ utils/readFileAsString.go | 15 +++++++++ 10 files changed, 313 insertions(+) create mode 100644 .gitignore create mode 100644 client/getUserUploads.go create mode 100644 getVidLinks.js create mode 100644 main.go create mode 100644 models/config.go create mode 100644 models/upload.go create mode 100644 models/upload_test.go create mode 100644 utils/downloadFile.go create mode 100644 utils/fileio.go create mode 100644 utils/readFileAsString.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..446b7fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +downloads +*.exe +tiktok-dl \ No newline at end of file diff --git a/client/getUserUploads.go b/client/getUserUploads.go new file mode 100644 index 0000000..901fcea --- /dev/null +++ b/client/getUserUploads.go @@ -0,0 +1,57 @@ +package client + +import ( + "context" + "github.com/chromedp/chromedp" + "io/ioutil" + "log" + "os" + "time" + + models "../models" + utils "../utils" +) + +// GetUserUploads - Get all uploads by user +func GetUserUploads(username string) []models.Upload { + dir, err := ioutil.TempDir("", "chromedp-example") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.DisableGPU, + chromedp.UserDataDir(dir), + chromedp.Flag("headless", false), + ) + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancel() + + ctx, cancel := chromedp.NewContext( + allocCtx, + chromedp.WithLogf(log.Printf), + ) + defer cancel() + + ctx, cancel = context.WithTimeout(ctx, 1500*time.Second) + defer cancel() + + var jsOutput string + err = chromedp.Run(ctx, + // Navigate to user's page + chromedp.Navigate(`https://www.tiktok.com/@`+username), + // Execute url grabber script + chromedp.EvaluateAsDevTools(utils.ReadFileAsString("getVidLinks.js"), &jsOutput), + // Wait until custom js finishes + chromedp.WaitVisible(`video_urls`), + // Grab url links from our element + chromedp.InnerHTML(`video_urls`, &jsOutput), + ) + if err != nil { + log.Fatal(err) + } + + return models.ParseUploads(jsOutput) +} diff --git a/getVidLinks.js b/getVidLinks.js new file mode 100644 index 0000000..a3cab70 --- /dev/null +++ b/getVidLinks.js @@ -0,0 +1,67 @@ +createVidUrlElement = function(videoSet) { + var videoArray = Object.entries(videoSet).map(x => { + return { + shareLink: x[1].shareLink, + url: x[0], + }; + }); + + var urlSetElement = document.createElement("video_urls"); + urlSetElement.innerText = JSON.stringify(videoArray); + document.getElementsByTagName("body")[0].appendChild(urlSetElement); +} + +buldVidUrlSet = function(finishCallback) { + var feedItem = document.getElementsByClassName("video-feed-item-wrapper")[0]; + feedItem.click(); + + var videoSet = {}; + var intervalID = window.setInterval(x => { + var players = document.getElementsByClassName("video-player"); + for (var i = 0; i < players.length; i++) { + var vidUrl = players[i].getAttribute("src"); + if(!videoSet[vidUrl]) { + var shareLink = document.querySelector(".copy-link-container > input").value; + videoSet[vidUrl] = { + shareLink: shareLink + }; + } + } + var arrowRight = document.querySelectorAll("div.video-card-modal > div > img.arrow-right")[0]; + if (arrowRight.classList.contains("disabled")) { + window.clearInterval(intervalID); + document.querySelector(".video-card-modal > div > div.close").click(); + finishCallback(videoSet); + } else { + arrowRight.click(); + } + }, 500); +}; + +scrollWhileNew = function(finishCallback) { + var state = { count: 0 }; + var intervalID = window.setInterval(x => { + var oldCount = state.count; + state.count = document.getElementsByClassName("video-feed-item").length; + if (oldCount !== state.count) { + window.scrollTo(0, document.body.scrollHeight); + } else { + window.clearInterval(intervalID); + finishCallback(); + } + }, 1000); +}; + +init = () => { + const newProto = navigator.__proto__; + delete newProto.webdriver; + navigator.__proto__ = newProto; + + window.setTimeout(x => { + window.scrollTo(0, document.body.scrollHeight); + window.setTimeout(x => buldVidUrlSet(createVidUrlElement), 2000); + }, 1000) +}; + +init(); +'script initialized' \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..09729fe --- /dev/null +++ b/main.go @@ -0,0 +1,30 @@ +package main + +import ( + client "./client" + models "./models" + utils "./utils" + "fmt" +) + +func main() { + models.GetConfig() + + username := models.Config.UserName + downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username) + uploads := client.GetUserUploads(username) + + utils.InitOutputDirectory(downloadDir) + + for _, upload := range uploads { + uploadID := upload.GetUploadID() + downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID) + + if utils.CheckIfExists(downloadPath) { + fmt.Println("Upload '" + uploadID + "' already downloaded, skipping") + continue + } + + utils.DownloadFile(downloadPath, upload.URL) + } +} diff --git a/models/config.go b/models/config.go new file mode 100644 index 0000000..7bcab99 --- /dev/null +++ b/models/config.go @@ -0,0 +1,28 @@ +package models + +import ( + "flag" + "fmt" + "os" +) + +// Config - Runtime configuration +var Config struct { + UserName string + OutputPath string +} + +// GetConfig - Returns Config object +func GetConfig() { + outputPath := flag.String("output", "./downloads", "Output path") + flag.Parse() + + args := flag.Args() + if len(args) < 1 { + fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME") + os.Exit(2) + } + + Config.UserName = flag.Args()[len(args)-1] + Config.OutputPath = *outputPath +} diff --git a/models/upload.go b/models/upload.go new file mode 100644 index 0000000..315ab92 --- /dev/null +++ b/models/upload.go @@ -0,0 +1,25 @@ +package models + +import ( + "encoding/json" + "strings" +) + +// Upload - Upload object +type Upload struct { + ShareLink string `json:"shareLink"` + URL string `json:"url"` +} + +// ParseUploads - Parses json uploads array +func ParseUploads(str string) []Upload { + var uploads []Upload + json.Unmarshal([]byte(str), &uploads) + return uploads +} + +// GetUploadID - Returns upload id +func (u Upload) GetUploadID() string { + parts := strings.Split(u.ShareLink, "/") + return parts[len(parts)-1] +} diff --git a/models/upload_test.go b/models/upload_test.go new file mode 100644 index 0000000..ca87801 --- /dev/null +++ b/models/upload_test.go @@ -0,0 +1,36 @@ +package models + +import "testing" + +// TestParseUploads - Test parsing +func TestParseUploads(t *testing.T) { + jsonStr := "[{\"shareLink\":\"some_share_link\", \"url\": \"some_url\"}]" + actual := ParseUploads(jsonStr) + + expectedLen := 1 + if len(actual) != expectedLen { + t.Errorf("Array len incorrect: Expected %d, but got %d", expectedLen, len(actual)) + } + + expectedShareLink := "some_share_link" + if actual[0].ShareLink != expectedShareLink { + t.Errorf("ShareLink is incorrect: Expected %s, but got %s", expectedShareLink, actual[0].ShareLink) + } + + expectedURL := "some_url" + if actual[0].URL != expectedURL { + t.Errorf("URL is incorrect: Expected %s, but got %s", expectedURL, actual[0].URL) + } +} + +func TestGetUploadID(t *testing.T) { + var upload Upload + upload.ShareLink = "http://pikami.org/some_thing/some_upload_id" + expected := "some_upload_id" + + actual := upload.GetUploadID() + + if actual != expected { + t.Errorf("UploadId is incorrect: Expected %s, but got %s", expected, actual) + } +} diff --git a/utils/downloadFile.go b/utils/downloadFile.go new file mode 100644 index 0000000..34d97c5 --- /dev/null +++ b/utils/downloadFile.go @@ -0,0 +1,31 @@ +package utils + +import ( + "io" + "net/http" + "os" +) + +// DownloadFile - Downloads content from `url` and stores it in `outputPath` +func DownloadFile(outputPath string, url string) { + // Get the data + resp, err := http.Get(url) + if err != nil { + panic(err) + } + defer resp.Body.Close() + + // Create the file + out, err := os.Create(outputPath) + if err != nil { + panic(err) + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + + if err != nil { + panic(err) + } +} diff --git a/utils/fileio.go b/utils/fileio.go new file mode 100644 index 0000000..e05ce7f --- /dev/null +++ b/utils/fileio.go @@ -0,0 +1,21 @@ +package utils + +import ( + "os" +) + +// CheckIfExists - Checks if file or directory exists +func CheckIfExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + + return true +} + +// InitOutputDirectory - Creates output directory +func InitOutputDirectory(path string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + os.MkdirAll(path, os.ModePerm) + } +} diff --git a/utils/readFileAsString.go b/utils/readFileAsString.go new file mode 100644 index 0000000..60f2e46 --- /dev/null +++ b/utils/readFileAsString.go @@ -0,0 +1,15 @@ +package utils + +import ( + "io/ioutil" + "log" +) + +// ReadFileAsString - Returns contents of given file +func ReadFileAsString(fileName string) string { + content, err := ioutil.ReadFile(fileName) + if err != nil { + log.Fatal(err) + } + return string(content) +}