From b6bb470064e1b461d6294d44a7bb9b6f627359a3 Mon Sep 17 00:00:00 2001 From: alexpin Date: Tue, 25 Feb 2020 01:01:10 +0200 Subject: [PATCH] formatter --- client/executeClientAction.go | 152 +++++++++++++++++----------------- client/getHashtagUploads.go | 18 ++-- client/getMusicUploads.go | 18 ++-- client/getUserUploads.go | 8 +- models/config/config.go | 86 +++++++++---------- utils/getHashtag.go | 14 ++-- workflows/downloadHashtag.go | 38 ++++----- workflows/downloadMusic.go | 38 ++++----- workflows/downloadUser.go | 2 +- workflows/downloadVideo.go | 50 +++++------ 10 files changed, 212 insertions(+), 212 deletions(-) diff --git a/client/executeClientAction.go b/client/executeClientAction.go index 6188103..94b02dc 100644 --- a/client/executeClientAction.go +++ b/client/executeClientAction.go @@ -1,102 +1,102 @@ package client import ( - "context" - "github.com/chromedp/chromedp" - "io/ioutil" - "log" - "os" - "time" + "context" + "github.com/chromedp/chromedp" + "io/ioutil" + "log" + "os" + "time" - config "../models/config" - utils "../utils" + config "../models/config" + utils "../utils" ) // GetMusicUploads - Get all uploads by given music func executeClientAction(url string, jsAction string) string { - dir, err := ioutil.TempDir("", "chromedp-example") - utils.CheckErr(err) - defer os.RemoveAll(dir) + dir, err := ioutil.TempDir("", "chromedp-example") + utils.CheckErr(err) + defer os.RemoveAll(dir) - opts := append(chromedp.DefaultExecAllocatorOptions[:], - chromedp.DisableGPU, - chromedp.UserDataDir(dir), - chromedp.Flag("headless", !config.Config.Debug), - ) + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.DisableGPU, + chromedp.UserDataDir(dir), + chromedp.Flag("headless", !config.Config.Debug), + ) - allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) - defer cancel() + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancel() - ctx, cancel := chromedp.NewContext( - allocCtx, - chromedp.WithLogf(log.Printf), - ) - defer cancel() + ctx, cancel := chromedp.NewContext( + allocCtx, + chromedp.WithLogf(log.Printf), + ) + defer cancel() - ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second) - defer cancel() + ctx, cancel = context.WithTimeout(ctx, time.Duration(config.Config.Deadline)*time.Second) + defer cancel() - var jsOutput string - jsOutput = runScrapeWithInfo(ctx, jsAction, url) + var jsOutput string + jsOutput = runScrapeWithInfo(ctx, jsAction, url) - return jsOutput + return jsOutput } func runScrapeQuiet(ctx context.Context, jsAction string, url string) string { - var jsOutput string - err := chromedp.Run(ctx, - // Navigate to user's page - chromedp.Navigate(url), - // Execute url grabber script - chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), - chromedp.EvaluateAsDevTools(jsAction, &jsOutput), - // Wait until custom js finishes - chromedp.WaitVisible(`video_urls`), - // Grab url links from our element - chromedp.InnerHTML(`video_urls`, &jsOutput), - ) - utils.CheckErr(err) - return jsOutput + var jsOutput string + err := chromedp.Run(ctx, + // Navigate to user's page + chromedp.Navigate(url), + // Execute url grabber script + chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), + chromedp.EvaluateAsDevTools(jsAction, &jsOutput), + // Wait until custom js finishes + chromedp.WaitVisible(`video_urls`), + // Grab url links from our element + chromedp.InnerHTML(`video_urls`, &jsOutput), + ) + utils.CheckErr(err) + return jsOutput } func runScrapeWithInfo(ctx context.Context, jsAction string, url string) string { - var jsOutput string - err := chromedp.Run(ctx, - // Navigate to user's page - chromedp.Navigate(url), - // Execute url grabber script - chromedp.WaitReady("video"), - chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), - chromedp.EvaluateAsDevTools(jsAction, &jsOutput), - ) - utils.CheckErr(err) + var jsOutput string + err := chromedp.Run(ctx, + // Navigate to user's page + chromedp.Navigate(url), + // Execute url grabber script + chromedp.WaitReady("video"), + chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), + chromedp.EvaluateAsDevTools(jsAction, &jsOutput), + ) + utils.CheckErr(err) - for { - err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)) - utils.CheckErr(err) - if jsOutput != "0" { - utils.Logf("\rPreloading... %s items have been founded.", jsOutput) - } else { - utils.Logf("\rPreloading...") - } + for { + err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.preloadCount.toString()", &jsOutput)) + utils.CheckErr(err) + if jsOutput != "0" { + utils.Logf("\rPreloading... %s items have been founded.", jsOutput) + } else { + utils.Logf("\rPreloading...") + } - err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)) - utils.CheckErr(err) - if jsOutput == "true" { - break - } + err = chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)) + utils.CheckErr(err) + if jsOutput == "true" { + break + } - time.Sleep(50 * time.Millisecond) - } + time.Sleep(50 * time.Millisecond) + } - utils.Log("\nRetrieving items...") - err = chromedp.Run(ctx, - // Wait until custom js finishes - chromedp.WaitVisible(`video_urls`), - // Grab url links from our element - chromedp.InnerHTML(`video_urls`, &jsOutput), - ) - utils.CheckErr(err) + utils.Log("\nRetrieving items...") + err = chromedp.Run(ctx, + // Wait until custom js finishes + chromedp.WaitVisible(`video_urls`), + // Grab url links from our element + chromedp.InnerHTML(`video_urls`, &jsOutput), + ) + utils.CheckErr(err) - return jsOutput + return jsOutput } diff --git a/client/getHashtagUploads.go b/client/getHashtagUploads.go index b3bd10c..c9df788 100644 --- a/client/getHashtagUploads.go +++ b/client/getHashtagUploads.go @@ -1,19 +1,19 @@ package client import ( - models "../models" - config "../models/config" - "fmt" + models "../models" + config "../models/config" + "fmt" ) // GetUserUploads - Get all uploads marked with given hashtag func GetHashtagUploads(hashtagURL string) []models.Upload { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) - actionOutput := executeClientAction(hashtagURL, jsMethod) - return models.ParseUploads(actionOutput) + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + actionOutput := executeClientAction(hashtagURL, jsMethod) + return models.ParseUploads(actionOutput) } func GetHashtagUploadsJson(hashtagURL string) string { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) - return executeClientAction(hashtagURL, jsMethod) -} \ No newline at end of file + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + return executeClientAction(hashtagURL, jsMethod) +} diff --git a/client/getMusicUploads.go b/client/getMusicUploads.go index 5880fa0..b6f6a72 100644 --- a/client/getMusicUploads.go +++ b/client/getMusicUploads.go @@ -1,19 +1,19 @@ package client import ( - models "../models" - config "../models/config" - "fmt" + models "../models" + config "../models/config" + "fmt" ) // GetMusicUploads - Get all uploads by given music func GetMusicUploads(url string) []models.Upload { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) - actionOutput := executeClientAction(url, jsMethod) - return models.ParseUploads(actionOutput) + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + actionOutput := executeClientAction(url, jsMethod) + return models.ParseUploads(actionOutput) } func GetMusicUploadsJson(url string) string { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) - return executeClientAction(url, jsMethod) -} \ No newline at end of file + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + return executeClientAction(url, jsMethod) +} diff --git a/client/getUserUploads.go b/client/getUserUploads.go index 7f38567..4fb7d07 100644 --- a/client/getUserUploads.go +++ b/client/getUserUploads.go @@ -3,17 +3,17 @@ package client import ( models "../models" config "../models/config" - "fmt" + "fmt" ) // GetUserUploads - Get all uploads by user func GetUserUploads(username string) []models.Upload { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) actionOutput := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod) return models.ParseUploads(actionOutput) } func GetUserUploadsJson(username string) string { - jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) + jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit) return executeClientAction(`https://www.tiktok.com/@`+username, jsMethod) -} \ No newline at end of file +} diff --git a/models/config/config.go b/models/config/config.go index 7495896..3eb4ea1 100644 --- a/models/config/config.go +++ b/models/config/config.go @@ -1,57 +1,57 @@ package config import ( - "flag" - "fmt" - "os" + "flag" + "fmt" + "os" ) // Config - Runtime configuration var Config struct { - URL string - OutputPath string - BatchFilePath string - Debug bool - MetaData bool - Quiet bool - Deadline int - Limit int - JSONOnly bool + URL string + OutputPath string + BatchFilePath string + Debug bool + MetaData bool + Quiet bool + Deadline int + Limit int + JSONOnly bool } // GetConfig - Returns Config object func GetConfig() { - outputPath := flag.String("output", "./downloads", "Output path") - batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.") - debug := flag.Bool("debug", false, "Enables debug mode") - metadata := flag.Bool("metadata", false, "Write video metadata to a .json file") - quiet := flag.Bool("quiet", false, "Supress output") - deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)") - limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)") - jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)") - flag.Parse() + outputPath := flag.String("output", "./downloads", "Output path") + batchFilePath := flag.String("batch-file", "", "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.") + debug := flag.Bool("debug", false, "Enables debug mode") + metadata := flag.Bool("metadata", false, "Write video metadata to a .json file") + quiet := flag.Bool("quiet", false, "Supress output") + deadline := flag.Int("deadline", 1500, "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)") + limit := flag.Int("limit", 0, "Sets the videos count limit (useful when there too many videos from the user or by hashtag)") + jsonOnly := flag.Bool("json", false, "Just get JSON data from scraper (without video downloading)") + flag.Parse() - args := flag.Args() - if len(args) < 1 && *batchFilePath == "" { - fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL") - fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt") - os.Exit(2) - } + args := flag.Args() + if len(args) < 1 && *batchFilePath == "" { + fmt.Println("Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL") + fmt.Println(" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt") + os.Exit(2) + } - if len(args) > 0 { - Config.URL = flag.Args()[len(args)-1] - } else { - Config.URL = "" - } - Config.OutputPath = *outputPath - Config.BatchFilePath = *batchFilePath - Config.Debug = *debug - Config.MetaData = *metadata - Config.Quiet = *quiet - if *jsonOnly { - Config.Quiet = true - } - Config.Deadline = *deadline - Config.Limit = *limit - Config.JSONOnly = *jsonOnly; + if len(args) > 0 { + Config.URL = flag.Args()[len(args)-1] + } else { + Config.URL = "" + } + Config.OutputPath = *outputPath + Config.BatchFilePath = *batchFilePath + Config.Debug = *debug + Config.MetaData = *metadata + Config.Quiet = *quiet + if *jsonOnly { + Config.Quiet = true + } + Config.Deadline = *deadline + Config.Limit = *limit + Config.JSONOnly = *jsonOnly } diff --git a/utils/getHashtag.go b/utils/getHashtag.go index d9c8ba3..11c7092 100644 --- a/utils/getHashtag.go +++ b/utils/getHashtag.go @@ -1,16 +1,16 @@ package utils import ( - res "../resources" - "fmt" - "strings" + res "../resources" + "fmt" + "strings" ) // GetHashtagFromURL - Get's tag name from passed url func GetHashtagFromURL(str string) string { - if match := strings.Contains(str, "/tag/"); match { - return strings.Split(str, "/tag/")[1] - } + if match := strings.Contains(str, "/tag/"); match { + return strings.Split(str, "/tag/")[1] + } - panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str)) + panic(fmt.Sprintf(res.ErrorCouldNotRecogniseURL, str)) } diff --git a/workflows/downloadHashtag.go b/workflows/downloadHashtag.go index a5332bf..3b4ca01 100644 --- a/workflows/downloadHashtag.go +++ b/workflows/downloadHashtag.go @@ -1,36 +1,36 @@ package workflows import ( - client "../client" - config "../models/config" - utils "../utils" - "fmt" - "strings" + client "../client" + config "../models/config" + utils "../utils" + "fmt" + "strings" ) // CanUseDownloadHashtag - Test's if this workflow can be used for parameter func CanUseDownloadHashtag(url string) bool { - match := strings.Contains(url, "/tag/") - return match + match := strings.Contains(url, "/tag/") + return match } // DownloadHashtag - Download videos marked with given hashtag func DownloadHashtag(url string) { - uploads := client.GetHashtagUploads(url) - uploadCount := len(uploads) - hashtag := utils.GetHashtagFromURL(url) - downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, hashtag) + uploads := client.GetHashtagUploads(url) + uploadCount := len(uploads) + hashtag := utils.GetHashtagFromURL(url) + downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, hashtag) - utils.InitOutputDirectory(downloadDir) + utils.InitOutputDirectory(downloadDir) - for index, upload := range uploads { - downloadVideo(upload, downloadDir) - utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) - } - utils.Log() + for index, upload := range uploads { + downloadVideo(upload, downloadDir) + utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) + } + utils.Log() } func GetHashtagJson(url string) { - uploads := client.GetHashtagUploads(url) - fmt.Printf("%s", uploads) + uploads := client.GetHashtagUploads(url) + fmt.Printf("%s", uploads) } diff --git a/workflows/downloadMusic.go b/workflows/downloadMusic.go index d6e2c8f..ef1491e 100644 --- a/workflows/downloadMusic.go +++ b/workflows/downloadMusic.go @@ -1,36 +1,36 @@ package workflows import ( - client "../client" - config "../models/config" - utils "../utils" - "fmt" - "regexp" + client "../client" + config "../models/config" + utils "../utils" + "fmt" + "regexp" ) // CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter func CanUseDownloadMusic(url string) bool { - match, _ := regexp.MatchString(".com\\/music\\/.+", url) - return match + match, _ := regexp.MatchString(".com\\/music\\/.+", url) + return match } // DownloadMusic - Download all videos by given music func DownloadMusic(url string) { - uploads := client.GetMusicUploads(url) - uploadCount := len(uploads) + uploads := client.GetMusicUploads(url) + uploadCount := len(uploads) - for index, upload := range uploads { - username := utils.GetUsernameFromString(upload.Uploader) - downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) + for index, upload := range uploads { + username := utils.GetUsernameFromString(upload.Uploader) + downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) - utils.InitOutputDirectory(downloadDir) - downloadVideo(upload, downloadDir) - utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) - } - utils.Log() + utils.InitOutputDirectory(downloadDir) + downloadVideo(upload, downloadDir) + utils.Logf("\r[%d/%d] Downloaded", index+1, uploadCount) + } + utils.Log() } func GetMusicJson(url string) { - uploads := client.GetMusicUploadsJson(url) - fmt.Printf("%s", uploads) + uploads := client.GetMusicUploadsJson(url) + fmt.Printf("%s", uploads) } diff --git a/workflows/downloadUser.go b/workflows/downloadUser.go index cd73373..0967c75 100644 --- a/workflows/downloadUser.go +++ b/workflows/downloadUser.go @@ -31,5 +31,5 @@ func DownloadUser(username string) { func GetUserVideosJson(username string) { uploads := client.GetUserUploadsJson(username) - fmt.Printf("%s", uploads) + fmt.Printf("%s", uploads) } diff --git a/workflows/downloadVideo.go b/workflows/downloadVideo.go index 990d330..e538db1 100644 --- a/workflows/downloadVideo.go +++ b/workflows/downloadVideo.go @@ -1,44 +1,44 @@ package workflows import ( - client "../client" - models "../models" - config "../models/config" - utils "../utils" - "fmt" - "regexp" + client "../client" + models "../models" + config "../models/config" + utils "../utils" + "fmt" + "regexp" ) // CanUseDownloadSingleVideo - Check's if DownloadSingleVideo can be used for parameter func CanUseDownloadSingleVideo(url string) bool { - match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url) - return match + match, _ := regexp.MatchString("\\/@.+\\/video\\/[0-9]+", url) + return match } // DownloadSingleVideo - Downloads single video func DownloadSingleVideo(url string) { - username := utils.GetUsernameFromString(url) - upload := client.GetVideoDetails(url) - downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) + username := utils.GetUsernameFromString(url) + upload := client.GetVideoDetails(url) + downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username) - utils.InitOutputDirectory(downloadDir) - downloadVideo(upload, downloadDir) - utils.Log("[1/1] Downloaded\n") + utils.InitOutputDirectory(downloadDir) + downloadVideo(upload, downloadDir) + utils.Log("[1/1] Downloaded\n") } // DownloadVideo - Downloads one video func downloadVideo(upload models.Upload, downloadDir string) { - uploadID := upload.GetUploadID() - downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID) + uploadID := upload.GetUploadID() + downloadPath := fmt.Sprintf("%s/%s.mp4", downloadDir, uploadID) - if utils.CheckIfExists(downloadPath) { - return - } + if utils.CheckIfExists(downloadPath) { + return + } - utils.DownloadFile(downloadPath, upload.URL) + utils.DownloadFile(downloadPath, upload.URL) - if config.Config.MetaData { - metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID) - upload.WriteToFile(metadataPath) - } -} \ No newline at end of file + if config.Config.MetaData { + metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID) + upload.WriteToFile(metadataPath) + } +}