diff --git a/README.md b/README.md index 75f59f1..6bc44bc 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ A simple tiktok video downloader written in go Download the executable from `https://github.com/pikami/tiktok-dl/releases`\ You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\ You can download single video by running `./tiktok-dl [Options] VIDEO_URL`\ +You can download all videos by music by running `./tiktok-dl [Options] MUSIC_URL`\ You can download items listed in a text file by running `./tiktok-dl [OPTIONS] -batch-file path/to/items.txt` ## Build instructions diff --git a/client/getMusicUploads.go b/client/getMusicUploads.go new file mode 100644 index 0000000..1ab1977 --- /dev/null +++ b/client/getMusicUploads.go @@ -0,0 +1,58 @@ +package client + +import ( + "context" + "github.com/chromedp/chromedp" + "io/ioutil" + "log" + "os" + "time" + + models "../models" + utils "../utils" +) + +// GetMusicUploads - Get all uploads by given music +func GetMusicUploads(url string) []models.Upload { + dir, err := ioutil.TempDir("", "chromedp-example") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.DisableGPU, + chromedp.UserDataDir(dir), + chromedp.Flag("headless", !models.Config.Debug), + ) + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer cancel() + + ctx, cancel := chromedp.NewContext( + allocCtx, + chromedp.WithLogf(log.Printf), + ) + defer cancel() + + ctx, cancel = context.WithTimeout(ctx, 1500*time.Second) + defer cancel() + + var jsOutput string + err = chromedp.Run(ctx, + // Navigate to user's page + chromedp.Navigate(url), + // Execute url grabber script + chromedp.EvaluateAsDevTools(utils.ReadFileAsString("scraper.js"), &jsOutput), + chromedp.EvaluateAsDevTools("bootstrapIteratingVideos()", &jsOutput), + // Wait until custom js finishes + chromedp.WaitVisible(`video_urls`), + // Grab url links from our element + chromedp.InnerHTML(`video_urls`, &jsOutput), + ) + if err != nil { + log.Fatal(err) + } + + return models.ParseUploads(jsOutput) +} diff --git a/main.go b/main.go index 262f46f..9b7a2f1 100644 --- a/main.go +++ b/main.go @@ -16,17 +16,5 @@ func main() { return } - // Single video - if workflows.CanUseDownloadSingleVideo(url) { - workflows.DownloadSingleVideo(url) - return - } - - // Tiktok user - if workflows.CanUseDownloadUser(url) { - workflows.DownloadUser(models.GetUsername()) - return - } - - panic("Could not recognise URL format") + workflows.StartWorkflowByParameter(url) } diff --git a/models/upload.go b/models/upload.go index f0ef37d..09e579d 100644 --- a/models/upload.go +++ b/models/upload.go @@ -12,6 +12,7 @@ type Upload struct { URL string `json:"url"` ShareLink string `json:"shareLink"` Caption string `json:"caption"` + Uploader string `json:"uploader"` Sound Sound `json:"sound"` } diff --git a/models/upload_test.go b/models/upload_test.go index 440c069..6b6dc4a 100644 --- a/models/upload_test.go +++ b/models/upload_test.go @@ -9,7 +9,7 @@ import ( func TestParseUploads(t *testing.T) { tu := testUtil.TestUtil{T: t} - jsonStr := "[{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}]" + jsonStr := "[{\"url\":\"some_url\",\"shareLink\":\"some_share_link\",\"caption\":\"some_caption\", \"uploader\": \"some.uploader\",\"sound\":{\"title\":\"some_title\",\"link\":\"some_link\"}}]" actual := ParseUploads(jsonStr) tu.AssertInt(len(actual), 1, "Array len") @@ -17,6 +17,7 @@ func TestParseUploads(t *testing.T) { tu.AssertString(actual[0].URL, "some_url", "URL") tu.AssertString(actual[0].Caption, "some_caption", "Caption") tu.AssertString(actual[0].ShareLink, "some_share_link", "ShareLink") + tu.AssertString(actual[0].Uploader, "some.uploader", "Uploader") tu.AssertString(actual[0].Sound.Link, "some_link", "Sound.Link") tu.AssertString(actual[0].Sound.Title, "some_title", "Sound.Title") diff --git a/scraper.js b/scraper.js index 2afad91..d174c2a 100644 --- a/scraper.js +++ b/scraper.js @@ -7,10 +7,12 @@ optStrings = { modalShareInput: '.copy-link-container > input', modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1', modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a', + modalUploader: '.user-username', videoPlayer: 'div.video-card-container > div > div > video', videoShareInput: 'div.content-container.border > div.copy-link-container > input', videoCaption: 'div.content-container.border > div.video-meta-info > h1', videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a', + videoUploader: '.user-username', }, classes: { feedVideoItem: 'video-feed-item-wrapper', @@ -56,6 +58,7 @@ getCurrentModalVideo = function() { var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value; var caption = document.querySelector(optStrings.selectors.modalCaption).textContent; var soundLink = document.querySelector(optStrings.selectors.modalSoundLink); + var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent; var soundHref = soundLink.getAttribute("href"); var soundText = soundLink.text; @@ -63,6 +66,7 @@ getCurrentModalVideo = function() { url: vidUrl, shareLink: shareLink, caption: caption, + uploader: uploader, sound: { title: soundText, link: soundHref, @@ -76,6 +80,7 @@ getCurrentVideo = function() { var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value; var caption = document.querySelector(optStrings.selectors.videoCaption).textContent; var soundLink = document.querySelector(optStrings.selectors.videoSoundLink); + var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent; var soundHref = soundLink.getAttribute("href"); var soundText = soundLink.text; @@ -83,6 +88,7 @@ getCurrentVideo = function() { url: vidUrl, shareLink: shareLink, caption: caption, + uploader: uploader, sound: { title: soundText, link: soundHref, diff --git a/workflows/downloadBatchFile.go b/workflows/downloadBatchFile.go index d4f3c18..989dc83 100644 --- a/workflows/downloadBatchFile.go +++ b/workflows/downloadBatchFile.go @@ -1,7 +1,6 @@ package workflows import ( - models "../models" utils "../utils" "fmt" ) @@ -25,17 +24,5 @@ func downloadItem(batchItem string) { return } - // Single video - if CanUseDownloadSingleVideo(batchItem) { - DownloadSingleVideo(batchItem) - return - } - - // Tiktok user - if CanUseDownloadUser(batchItem) { - DownloadUser(models.GetUsernameFromString(batchItem)) - return - } - - panic(fmt.Sprintf("Could not recognise URL format of string %s", batchItem)) + StartWorkflowByParameter(batchItem) } diff --git a/workflows/downloadMusic.go b/workflows/downloadMusic.go new file mode 100644 index 0000000..09cba06 --- /dev/null +++ b/workflows/downloadMusic.go @@ -0,0 +1,28 @@ +package workflows + +import ( + client "../client" + models "../models" + utils "../utils" + "fmt" + "regexp" +) + +// CanUseDownloadMusic - Check's if DownloadMusic can be used for parameter +func CanUseDownloadMusic(url string) bool { + match, _ := regexp.MatchString(".com\\/music\\/.+", url) + return match +} + +// DownloadMusic - Download all videos by given music +func DownloadMusic(url string) { + uploads := client.GetMusicUploads(url) + + for _, upload := range uploads { + username := models.GetUsernameFromString(upload.Uploader) + downloadDir := fmt.Sprintf("%s/%s", models.Config.OutputPath, username) + + utils.InitOutputDirectory(downloadDir) + downloadVideo(upload, downloadDir) + } +} diff --git a/workflows/startWorkflowByParameter.go b/workflows/startWorkflowByParameter.go new file mode 100644 index 0000000..10666dc --- /dev/null +++ b/workflows/startWorkflowByParameter.go @@ -0,0 +1,30 @@ +package workflows + +import ( + models "../models" + "fmt" +) + +// StartWorkflowByParameter - Start needed workflow by given parameter +func StartWorkflowByParameter(url string) { + + // Music + if CanUseDownloadMusic(url) { + DownloadMusic(url) + return + } + + // Single video + if CanUseDownloadSingleVideo(url) { + DownloadSingleVideo(url) + return + } + + // Tiktok user + if CanUseDownloadUser(url) { + DownloadUser(models.GetUsername()) + return + } + + panic(fmt.Sprintf("Could not recognise URL format of string %s", url)) +}