mirror of https://github.com/pikami/tiktok-dl.git
Merge branch 'master' into master
This commit is contained in:
commit
e77c904f89
|
@ -2,10 +2,12 @@ package client
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/chromedp/chromedp"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
config "../models/config"
|
||||
|
@ -13,9 +15,11 @@ import (
|
|||
)
|
||||
|
||||
// GetMusicUploads - Get all uploads by given music
|
||||
func executeClientAction(url string, jsAction string) string {
|
||||
func executeClientAction(url string, jsAction string) (string, error) {
|
||||
dir, err := ioutil.TempDir("", "chromedp-example")
|
||||
utils.CheckErr(err)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
|
@ -36,15 +40,16 @@ func executeClientAction(url string, jsAction string) string {
|
|||
ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var jsOutput string
|
||||
jsOutput = runScrapeWithInfo(ctx, jsAction, url)
|
||||
|
||||
return jsOutput
|
||||
jsOutput, err := runScrapeWithInfo(ctx, jsAction, url)
|
||||
if strings.HasPrefix(jsOutput, "\"ERR:") {
|
||||
err = errors.New(jsOutput)
|
||||
}
|
||||
return jsOutput, err
|
||||
}
|
||||
|
||||
func runScrapeQuiet(ctx context.Context, jsAction string, url string) string {
|
||||
func runScrapeQuiet(ctx context.Context, jsAction string, url string) (string, error) {
|
||||
var jsOutput string
|
||||
err := chromedp.Run(ctx,
|
||||
if err := chromedp.Run(ctx,
|
||||
// Navigate to user's page
|
||||
chromedp.Navigate(url),
|
||||
// Execute url grabber script
|
||||
|
@ -54,33 +59,40 @@ func runScrapeQuiet(ctx context.Context, jsAction string, url string) string {
|
|||
chromedp.WaitVisible(`video_urls`),
|
||||
// Grab url links from our element
|
||||
chromedp.InnerHTML(`video_urls`, &jsOutput),
|
||||
)
|
||||
utils.CheckErr(err)
|
||||
return jsOutput
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string {
|
||||
return jsOutput, nil
|
||||
}
|
||||
|
||||
func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string, error) {
|
||||
var jsOutput string
|
||||
err := chromedp.Run(ctx,
|
||||
if err := chromedp.Run(ctx,
|
||||
// Navigate to user's page
|
||||
chromedp.Navigate(url),
|
||||
// Execute url grabber script
|
||||
chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput),
|
||||
chromedp.EvaluateAsDevTools(jsAction, &jsOutput),
|
||||
)
|
||||
utils.CheckErr(err)
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
for {
|
||||
err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput))
|
||||
utils.CheckErr(err)
|
||||
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if jsOutput != "0" {
|
||||
utils.Logf("\rPreloading... %s items have been founded.", jsOutput)
|
||||
} else {
|
||||
utils.Logf("\rPreloading...")
|
||||
}
|
||||
|
||||
err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput))
|
||||
utils.CheckErr(err)
|
||||
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if jsOutput == "true" {
|
||||
break
|
||||
}
|
||||
|
@ -89,13 +101,14 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string
|
|||
}
|
||||
|
||||
utils.Log("\nRetrieving items...")
|
||||
err = chromedp.Run(ctx,
|
||||
if err := chromedp.Run(ctx,
|
||||
// Wait until custom js finishes
|
||||
chromedp.WaitVisible(`video_urls`),
|
||||
// Grab url links from our element
|
||||
chromedp.InnerHTML(`video_urls`, &jsOutput),
|
||||
)
|
||||
utils.CheckErr(err)
|
||||
|
||||
return jsOutput
|
||||
); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return jsOutput, nil
|
||||
}
|
||||
|
|
|
@ -7,10 +7,13 @@ import (
|
|||
)
|
||||
|
||||
// GetMusicUploads - Get all uploads by given music
|
||||
func GetMusicUploads(url string) []models.Upload {
|
||||
func GetMusicUploads(url string) ([]models.Upload, error) {
|
||||
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
|
||||
actionOutput := executeClientAction(url, jsMethod)
|
||||
return models.ParseUploads(actionOutput)
|
||||
actionOutput, err := executeClientAction(url, jsMethod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return models.ParseUploads(actionOutput), nil
|
||||
}
|
||||
|
||||
func GetMusicUploadsJson(url string) string {
|
||||
|
|
|
@ -7,10 +7,13 @@ import (
|
|||
)
|
||||
|
||||
// GetUserUploads - Get all uploads by user
|
||||
func GetUserUploads(username string) []models.Upload {
|
||||
func GetUserUploads(username string) ([]models.Upload, error) {
|
||||
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
|
||||
actionOutput := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
|
||||
return models.ParseUploads(actionOutput)
|
||||
actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return models.ParseUploads(actionOutput), nil
|
||||
}
|
||||
|
||||
func GetUserUploadsJson(username string) string {
|
||||
|
|
|
@ -5,7 +5,10 @@ import (
|
|||
)
|
||||
|
||||
// GetVideoDetails - returns details of video
|
||||
func GetVideoDetails(videoURL string) models.Upload {
|
||||
actionOutput := executeClientAction(videoURL, "bootstrapGetCurrentVideo()")
|
||||
return models.ParseUpload(actionOutput)
|
||||
func GetVideoDetails(videoURL string) (models.Upload, error) {
|
||||
actionOutput, err := executeClientAction(videoURL, "bootstrapGetCurrentVideo()")
|
||||
if err != nil {
|
||||
return models.Upload{}, err
|
||||
}
|
||||
return models.ParseUpload(actionOutput), nil
|
||||
}
|
||||
|
|
|
@ -6,5 +6,8 @@ var ErrorCouldNotSerializeJSON = "Could not serialize json for video: %s\n"
|
|||
// ErrorCouldNotRecogniseURL -
|
||||
var ErrorCouldNotRecogniseURL = "Could not recognise URL format of string %s"
|
||||
|
||||
// ErrorCouldNotGetUserUploads -
|
||||
var ErrorCouldNotGetUserUploads = "Failed to get user uploads: %s\n"
|
||||
|
||||
// ErrorPathNotFound -
|
||||
var ErrorPathNotFound = "File path %s not found."
|
||||
|
|
24
scraper.js
24
scraper.js
|
@ -17,6 +17,7 @@ optStrings = {
|
|||
classes: {
|
||||
feedVideoItem: 'video-feed-item-wrapper',
|
||||
modalCloseDisabled: 'disabled',
|
||||
titleMessage: 'title',
|
||||
},
|
||||
tags: {
|
||||
resultTag: 'video_urls',
|
||||
|
@ -25,6 +26,11 @@ optStrings = {
|
|||
attributes: {
|
||||
src: "src",
|
||||
},
|
||||
tiktokMessages: [
|
||||
"Couldn't find this account",
|
||||
"No videos yet",
|
||||
"Video currently unavailable",
|
||||
],
|
||||
};
|
||||
|
||||
currentState = {
|
||||
|
@ -33,6 +39,19 @@ currentState = {
|
|||
limit: 0
|
||||
};
|
||||
|
||||
checkForErrors = function() {
|
||||
var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
|
||||
debugger;
|
||||
if (titles && titles.length) {
|
||||
var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
|
||||
if (error) {
|
||||
createVidUrlElement("ERR: " + error);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
createVidUrlElement = function(outputObj) {
|
||||
var urlSetElement = document.createElement(optStrings.tags.resultTag);
|
||||
urlSetElement.innerText = JSON.stringify(outputObj);
|
||||
|
@ -88,6 +107,7 @@ getCurrentModalVideo = function() {
|
|||
};
|
||||
|
||||
getCurrentVideo = function() {
|
||||
if(checkForErrors()) return;
|
||||
var player = document.querySelector(optStrings.selectors.videoPlayer);
|
||||
var vidUrl = player.getAttribute(optStrings.attributes.src);
|
||||
var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
|
||||
|
@ -120,6 +140,10 @@ scrollWhileNew = function(finishCallback) {
|
|||
window.clearInterval(intervalID);
|
||||
}
|
||||
}
|
||||
if(checkForErrors()) {
|
||||
window.clearInterval(intervalID);
|
||||
return;
|
||||
}
|
||||
if (oldCount !== state.count) {
|
||||
currentState.preloadCount = state.count;
|
||||
window.scrollTo(0, document.body.scrollHeight);
|
||||
|
|
|
@ -3,6 +3,7 @@ package utils
|
|||
import (
|
||||
config "../models/config"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Log - Write to std out
|
||||
|
@ -23,3 +24,8 @@ func Logf(format string, a ...interface{}) {
|
|||
func LogFatal(format string, a ...interface{}) {
|
||||
panic(fmt.Sprintf(format, a...))
|
||||
}
|
||||
|
||||
// LogErr - Write error
|
||||
func LogErr(format string, a ...interface{}) {
|
||||
fmt.Fprintf(os.Stderr, format, a...)
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ package workflows
|
|||
import (
|
||||
client "../client"
|
||||
config "../models/config"
|
||||
res "../resources"
|
||||
utils "../utils"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
@ -16,7 +17,11 @@ func CanUseDownloadMusic(url string) bool {
|
|||
|
||||
// DownloadMusic - Download all videos by given music
|
||||
func DownloadMusic(url string) {
|
||||
uploads := client.GetMusicUploads(url)
|
||||
uploads, err := client.GetMusicUploads(url)
|
||||
if err != nil {
|
||||
utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
||||
return
|
||||
}
|
||||
uploadCount := len(uploads)
|
||||
|
||||
for index, upload := range uploads {
|
||||
|
|
|
@ -3,20 +3,27 @@ package workflows
|
|||
import (
|
||||
client "../client"
|
||||
config "../models/config"
|
||||
res "../resources"
|
||||
utils "../utils"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CanUseDownloadUser - Test's if this workflow can be used for parameter
|
||||
func CanUseDownloadUser(url string) bool {
|
||||
match := strings.Contains(url, "/")
|
||||
return !match
|
||||
isURL := strings.Contains(url, "/")
|
||||
match, _ := regexp.MatchString(".+com\\/@[^\\/]+", url)
|
||||
return !isURL || match
|
||||
}
|
||||
|
||||
// DownloadUser - Download all user's videos
|
||||
func DownloadUser(username string) {
|
||||
uploads := client.GetUserUploads(username)
|
||||
uploads, err := client.GetUserUploads(username)
|
||||
if err != nil {
|
||||
utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
||||
return
|
||||
}
|
||||
uploadCount := len(uploads)
|
||||
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
|
||||
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
client "../client"
|
||||
models "../models"
|
||||
config "../models/config"
|
||||
res "../resources"
|
||||
utils "../utils"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
@ -18,7 +19,11 @@ func CanUseDownloadSingleVideo(url string) bool {
|
|||
// DownloadSingleVideo - Downloads single video
|
||||
func DownloadSingleVideo(url string) {
|
||||
username := utils.GetUsernameFromString(url)
|
||||
upload := client.GetVideoDetails(url)
|
||||
upload, err := client.GetVideoDetails(url)
|
||||
if err != nil {
|
||||
utils.LogErr(res.ErrorCouldNotGetUserUploads, err.Error())
|
||||
return
|
||||
}
|
||||
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
|
||||
|
||||
utils.InitOutputDirectory(downloadDir)
|
||||
|
|
Loading…
Reference in New Issue