Compare commits

...

4 Commits

Author SHA1 Message Date
Pijus Kamandulis
e2e4ba0d4b Added ability to download from scrapeData (json) 2020-08-16 18:56:30 +03:00
Pijus Kamandulis
7a25f521cb Updated scraper.js 2020-08-16 01:30:48 +03:00
Pijus Kamandulis
e9715c0d40
Update README.md 2020-05-02 09:48:13 +03:00
Pijus Kamandulis
a70dc4cf04
Update README.md 2020-05-02 09:46:43 +03:00
9 changed files with 199 additions and 126 deletions

View File

@ -5,6 +5,10 @@
A simple tiktok video downloader written in go A simple tiktok video downloader written in go
```diff
- This tool is not working currenly, I will revive it in the future when I have some free time
```
## Basic usage examples ## Basic usage examples
Download the executable from `https://github.com/pikami/tiktok-dl/releases`\ Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\ You can download all videos from user by running `./tiktok-dl TIKTOK_USERNAME`\

View File

@ -34,7 +34,8 @@ var (
"ItemsFoundInArchive": "%d items, found in archive. Skipping...\n", "ItemsFoundInArchive": "%d items, found in archive. Skipping...\n",
"Downloaded": "\r[%d/%d] Downloaded", "Downloaded": "\r[%d/%d] Downloaded",
"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" + "UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
" or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt", " or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n" +
" or: tiktok-dl [OPTIONS] -scraped-data path/to/data.json",
}, },
}, },
resource{ resource{
@ -49,6 +50,10 @@ var (
"BatchFlag": "batch-file", "BatchFlag": "batch-file",
"BatchDefault": "", "BatchDefault": "",
"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.", "BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
// ScrapedData
"ScrapedDataFlag": "scraped-data",
"ScrapedDataDefault": "",
"ScrapedDataDescription": "Download videos from scrape file (json format)",
// Archive // Archive
"ArchiveFlag": "archive", "ArchiveFlag": "archive",
"ArchiveDefault": "", "ArchiveDefault": "",

View File

@ -9,6 +9,7 @@ func main() {
config.GetConfig() config.GetConfig()
url := config.Config.URL url := config.Config.URL
batchFilePath := config.Config.BatchFilePath batchFilePath := config.Config.BatchFilePath
scrapedDataFilePath := config.Config.ScrapedDataFilePath
// Batch file // Batch file
if workflows.CanUseDownloadBatchFile(batchFilePath) { if workflows.CanUseDownloadBatchFile(batchFilePath) {
@ -16,5 +17,11 @@ func main() {
return return
} }
// Scraped data file
if workflows.CanUseDownloadScrapedData(scrapedDataFilePath) {
workflows.DownloadScrapedData(scrapedDataFilePath)
return
}
workflows.StartWorkflowByParameter(url) workflows.StartWorkflowByParameter(url)
} }

View File

@ -11,23 +11,25 @@ import (
// Config - Runtime configuration // Config - Runtime configuration
var Config struct { var Config struct {
URL string URL string
OutputPath string OutputPath string
BatchFilePath string BatchFilePath string
ArchiveFilePath string ScrapedDataFilePath string
FailLogFilePath string ArchiveFilePath string
Debug bool FailLogFilePath string
MetaData bool Debug bool
Quiet bool MetaData bool
JSONOnly bool Quiet bool
Deadline int JSONOnly bool
Limit int Deadline int
Limit int
} }
// GetConfig - Returns Config object // GetConfig - Returns Config object
func GetConfig() { func GetConfig() {
outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription) outputPath := flag.String(res.OutputFlag, res.OutputDefault, res.OutputDescription)
batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription) batchFilePath := flag.String(res.BatchFlag, res.BatchDefault, res.BatchDescription)
scrapedDataFilePath := flag.String(res.ScrapedDataFlag, res.ScrapedDataDefault, res.ScrapedDataDescription)
archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription) archive := flag.String(res.ArchiveFlag, res.ArchiveDefault, res.ArchiveDescription)
failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription) failLogPath := flag.String(res.FailLogFlag, res.FailLogDefault, res.FailLogDescription)
debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription) debug := flag.Bool(res.DebugFlag, parseBool(res.DebugDefault), res.DebugDescription)
@ -39,7 +41,7 @@ func GetConfig() {
flag.Parse() flag.Parse()
args := flag.Args() args := flag.Args()
if len(args) < 1 && *batchFilePath == "" { if len(args) < 1 && *batchFilePath == "" && *scrapedDataFilePath == "" {
fmt.Println(res.UsageLine) fmt.Println(res.UsageLine)
os.Exit(2) os.Exit(2)
} }
@ -51,6 +53,7 @@ func GetConfig() {
} }
Config.OutputPath = *outputPath Config.OutputPath = *outputPath
Config.BatchFilePath = *batchFilePath Config.BatchFilePath = *batchFilePath
Config.ScrapedDataFilePath = *scrapedDataFilePath
Config.ArchiveFilePath = *archive Config.ArchiveFilePath = *archive
Config.FailLogFilePath = *failLogPath Config.FailLogFilePath = *failLogPath
Config.Debug = *debug Config.Debug = *debug

View File

@ -3,92 +3,101 @@
// Check `/generator/resources.go` to change generated content // Check `/generator/resources.go` to change generated content
package resources package resources
//BatchDescription - //FailLogDefault -
var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored." var FailLogDefault = ""
//LimitDefault -
var LimitDefault = "0"
//DebugDescription -
var DebugDescription = "Enables debug mode"
//QuietDescription - //QuietDescription -
var QuietDescription = "Suppress output" var QuietDescription = "Suppress output"
//MetadataDefault - //ScrapedDataDefault -
var MetadataDefault = "false" var ScrapedDataDefault = ""
//LimitDescription -
var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
//OutputDescription -
var OutputDescription = "Output path"
//BatchDefault -
var BatchDefault = ""
//FailLogDefault -
var FailLogDefault = ""
//MetadataFlag -
var MetadataFlag = "metadata"
//ArchiveFlag -
var ArchiveFlag = "archive"
//MetadataDescription -
var MetadataDescription = "Write video metadata to a .json file"
//QuietDefault -
var QuietDefault = "false"
//DeadlineDefault -
var DeadlineDefault = "1500"
//JsonFlag -
var JsonFlag = "json"
//JsonDefault -
var JsonDefault = "false"
//DeadlineDescription -
var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
//OutputFlag -
var OutputFlag = "output"
//OutputDefault -
var OutputDefault = "./downloads"
//FailLogFlag -
var FailLogFlag = "fail-log"
//DebugDefault -
var DebugDefault = "false"
//ArchiveDefault -
var ArchiveDefault = ""
//FailLogDescription -
var FailLogDescription = "Write failed items to log file"
//BatchFlag -
var BatchFlag = "batch-file"
//QuietFlag -
var QuietFlag = "quiet"
//DeadlineFlag -
var DeadlineFlag = "deadline"
//LimitFlag -
var LimitFlag = "limit"
//ArchiveDescription - //ArchiveDescription -
var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it." var ArchiveDescription = "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it."
//FailLogFlag -
var FailLogFlag = "fail-log"
//QuietDefault -
var QuietDefault = "false"
//JsonFlag -
var JsonFlag = "json"
//ScrapedDataFlag -
var ScrapedDataFlag = "scraped-data"
//DebugFlag - //DebugFlag -
var DebugFlag = "debug" var DebugFlag = "debug"
//MetadataDefault -
var MetadataDefault = "false"
//DeadlineFlag -
var DeadlineFlag = "deadline"
//OutputFlag -
var OutputFlag = "output"
//BatchFlag -
var BatchFlag = "batch-file"
//ArchiveFlag -
var ArchiveFlag = "archive"
//ArchiveDefault -
var ArchiveDefault = ""
//JsonDefault -
var JsonDefault = "false"
//JsonDescription - //JsonDescription -
var JsonDescription = "Just get JSON data from scraper (without video downloading)" var JsonDescription = "Just get JSON data from scraper (without video downloading)"
//DeadlineDescription -
var DeadlineDescription = "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)"
//OutputDefault -
var OutputDefault = "./downloads"
//OutputDescription -
var OutputDescription = "Output path"
//BatchDescription -
var BatchDescription = "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored."
//LimitFlag -
var LimitFlag = "limit"
//LimitDescription -
var LimitDescription = "Sets the videos count limit (useful when there too many videos from the user or by hashtag)"
//ScrapedDataDescription -
var ScrapedDataDescription = "Download videos from scrape file (json format)"
//DebugDescription -
var DebugDescription = "Enables debug mode"
//MetadataFlag -
var MetadataFlag = "metadata"
//BatchDefault -
var BatchDefault = ""
//MetadataDescription -
var MetadataDescription = "Write video metadata to a .json file"
//LimitDefault -
var LimitDefault = "0"
//DeadlineDefault -
var DeadlineDefault = "1500"
//FailLogDescription -
var FailLogDescription = "Write failed items to log file"
//DebugDefault -
var DebugDefault = "false"
//QuietFlag -
var QuietFlag = "quiet"

View File

@ -19,4 +19,4 @@ var ItemsFoundInArchive = "%d items, found in archive. Skipping...\n"
var Downloaded = "\r[%d/%d] Downloaded" var Downloaded = "\r[%d/%d] Downloaded"
//UsageLine - //UsageLine -
var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt" var UsageLine = "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n or: tiktok-dl [OPTIONS] -batch-file path/to/users.txt\n or: tiktok-dl [OPTIONS] -scraped-data path/to/data.json"

View File

@ -3,8 +3,8 @@
// Check `/generator/resources.go` to change generated content // Check `/generator/resources.go` to change generated content
package resources package resources
//ScraperScript -
var ScraperScript = "optStrings={selectors:{feedLoading:\"div.tiktok-loading.feed-loading\",modalArrowRight:\"div.video-card-modal > div > img.arrow-right\",modalClose:\".video-card-modal > div > div.close\",modalPlayer:\"div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video\",modalShareInput:\".copy-link-container > input\",modalCaption:\"div.video-card-big > div.content-container > div.video-meta-info > h1\",modalSoundLink:\"div.content-container > div.video-meta-info > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > div > video\",videoShareInput:\"div.content-container.border > div.copy-link-container > input\",videoCaption:\"div.content-container.border > div.video-meta-info > h1\",videoSoundLink:\"div.content-container.border > div.video-meta-info > h2.music-info > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var i=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];i.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):i.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=document.querySelector(optStrings.selectors.modalShareInput).value,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),i=document.querySelector(optStrings.selectors.modalUploader).textContent,n=o.getAttribute(\"href\");return{url:e,shareLink:t,caption:r,uploader:i,sound:{title:o.text,link:n}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=document.querySelector(optStrings.selectors.videoShareInput).value,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),i=document.querySelector(optStrings.selectors.videoUploader).textContent,n=o.getAttribute(\"href\");return{url:e,shareLink:t,caption:r,uploader:i,sound:{title:o.text,link:n}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var i=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(i!==t.count)currentState.preloadCount=t.count;else{if(document.querySelector(optStrings.selectors.feedLoading))return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"
//ScraperPath - //ScraperPath -
var ScraperPath = "scraper.js" var ScraperPath = "scraper.js"
//ScraperScript -
var ScraperScript = "optStrings={selectors:{feedLoading:\".tiktok-ui-loading-container\",modalArrowRight:\"div > div.video-card-container > img.arrow-right\",modalClose:\"div > div.video-card-container > img.control-icon.close\",modalPlayer:\"div.video-card-container > div.video-card-browse > video\",modalCaption:\"div.content-container > div.video-infos-container > h1\",modalSoundLink:\"div.content-container > div.video-infos-container > h2.music-info > a\",modalUploader:\".user-username\",videoPlayer:\"div.video-card-container > div > video\",videoCaption:\"div.content-container > div.video-infos-container > h1\",videoSoundLink:\"div.content-container > div.video-infos-container > h2 > a\",videoUploader:\".user-username\"},classes:{feedVideoItem:\"video-feed-item-wrapper\",modalCloseDisabled:\"disabled\",titleMessage:\"title\"},tags:{resultTag:\"video_urls\",resultParentTag:\"body\"},attributes:{src:\"src\"},tiktokMessages:[\"Couldn't find this account\",\"No videos yet\",\"Video currently unavailable\"]},currentState={preloadCount:0,finished:!1,limit:0},checkForErrors=function(){var e=document.getElementsByClassName(optStrings.classes.titleMessage);if(e&&e.length){var t=Array.from(e).find(e=>optStrings.tiktokMessages.includes(e.textContent)).textContent;if(t)return createVidUrlElement(\"ERR: \"+t),!0}return!1},createVidUrlElement=function(e){var t=document.createElement(optStrings.tags.resultTag);t.innerText=JSON.stringify(e),document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(t),currentState.finished=!0},buldVidUrlArray=function(e){document.getElementsByClassName(optStrings.classes.feedVideoItem)[0].click();var t=[],r=window.setInterval(o=>{t.push(getCurrentModalVideo()),currentState.limit>0&&t.length>=currentState.limit&&(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t));var n=document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];!n||n.classList.contains(optStrings.classes.modalCloseDisabled)?(window.clearInterval(r),document.querySelector(optStrings.selectors.modalClose).click(),e(t)):n.click()},20)},getCurrentModalVideo=function(){var e=document.querySelector(optStrings.selectors.modalPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.modalCaption).textContent,o=document.querySelector(optStrings.selectors.modalSoundLink),n=document.querySelector(optStrings.selectors.modalUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}},getCurrentVideo=function(){if(!checkForErrors()){var e=document.querySelector(optStrings.selectors.videoPlayer).getAttribute(optStrings.attributes.src),t=window.location.href,r=document.querySelector(optStrings.selectors.videoCaption).textContent,o=document.querySelector(optStrings.selectors.videoSoundLink),n=document.querySelector(optStrings.selectors.videoUploader).textContent,i=o?o.getAttribute(\"href\"):\"\";return{url:e,shareLink:t,caption:r,uploader:n,sound:{title:o?o.text:\"\",link:i}}}},scrollBottom=()=>window.scrollTo(0,document.body.scrollHeight),scrollWhileNew=function(e){var t={count:0},r=window.setInterval(o=>{scrollBottom();var n=t.count;if(t.count=document.getElementsByClassName(optStrings.classes.feedVideoItem).length,currentState.limit>0&&(currentState.preloadCount>=currentState.limit||t.count>=currentState.limit)&&(e(createVidUrlElement),window.clearInterval(r)),checkForErrors())window.clearInterval(r);else if(0!=t.count)if(n!==t.count)currentState.preloadCount=t.count;else{if(isLoading())return;window.clearInterval(r),e(createVidUrlElement)}},1e3)},isLoading=function(){var e=document.querySelector(optStrings.selectors.feedLoading);return e&&0!=e.getClientRects().length},bootstrapIteratingVideos=function(e){return currentState.limit=e,scrollWhileNew(buldVidUrlArray),\"bootstrapIteratingVideos\"},bootstrapGetCurrentVideo=function(){var e=getCurrentVideo();return createVidUrlElement(e),\"bootstrapGetCurrentVideo\"},init=()=>{const e=navigator.__proto__;return delete e.webdriver,navigator.__proto__=e,\"script initialized\"},init();"

View File

@ -1,17 +1,15 @@
optStrings = { optStrings = {
selectors: { selectors: {
feedLoading: 'div.tiktok-loading.feed-loading', feedLoading: '.tiktok-ui-loading-container',
modalArrowRight: 'div.video-card-modal > div > img.arrow-right', modalArrowRight: 'div > div.video-card-container > img.arrow-right',
modalClose: '.video-card-modal > div > div.close', modalClose: 'div > div.video-card-container > img.control-icon.close',
modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video', modalPlayer: 'div.video-card-container > div.video-card-browse > video',
modalShareInput: '.copy-link-container > input', modalCaption: 'div.content-container > div.video-infos-container > h1',
modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1', modalSoundLink: 'div.content-container > div.video-infos-container > h2.music-info > a',
modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a',
modalUploader: '.user-username', modalUploader: '.user-username',
videoPlayer: 'div.video-card-container > div > div > video', videoPlayer: 'div.video-card-container > div > video',
videoShareInput: 'div.content-container.border > div.copy-link-container > input', videoCaption: 'div.content-container > div.video-infos-container > h1',
videoCaption: 'div.content-container.border > div.video-meta-info > h1', videoSoundLink: 'div.content-container > div.video-infos-container > h2 > a',
videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a',
videoUploader: '.user-username', videoUploader: '.user-username',
}, },
classes: { classes: {
@ -39,9 +37,9 @@ currentState = {
limit: 0 limit: 0
}; };
checkForErrors = function() { checkForErrors = function () {
var titles = document.getElementsByClassName(optStrings.classes.titleMessage); var titles = document.getElementsByClassName(optStrings.classes.titleMessage);
debugger; //debugger;
if (titles && titles.length) { if (titles && titles.length) {
var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent; var error = Array.from(titles).find(x => optStrings.tiktokMessages.includes(x.textContent)).textContent;
if (error) { if (error) {
@ -52,21 +50,21 @@ checkForErrors = function() {
return false; return false;
}; };
createVidUrlElement = function(outputObj) { createVidUrlElement = function (outputObj) {
var urlSetElement = document.createElement(optStrings.tags.resultTag); var urlSetElement = document.createElement(optStrings.tags.resultTag);
urlSetElement.innerText = JSON.stringify(outputObj); urlSetElement.innerText = JSON.stringify(outputObj);
document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement); document.getElementsByTagName(optStrings.tags.resultParentTag)[0].appendChild(urlSetElement);
currentState.finished = true; currentState.finished = true;
}; };
buldVidUrlArray = function(finishCallback) { buldVidUrlArray = function (finishCallback) {
var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0]; var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
feedItem.click(); feedItem.click();
var videoArray = []; var videoArray = [];
var intervalID = window.setInterval(x => { var intervalID = window.setInterval(x => {
videoArray.push(getCurrentModalVideo()); videoArray.push(getCurrentModalVideo());
if(currentState.limit > 0) { if (currentState.limit > 0) {
if (videoArray.length >= currentState.limit) { if (videoArray.length >= currentState.limit) {
window.clearInterval(intervalID); window.clearInterval(intervalID);
document.querySelector(optStrings.selectors.modalClose).click(); document.querySelector(optStrings.selectors.modalClose).click();
@ -74,7 +72,7 @@ buldVidUrlArray = function(finishCallback) {
} }
} }
var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0]; var arrowRight = document.querySelectorAll(optStrings.selectors.modalArrowRight)[0];
if (arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) { if (!arrowRight || arrowRight.classList.contains(optStrings.classes.modalCloseDisabled)) {
window.clearInterval(intervalID); window.clearInterval(intervalID);
document.querySelector(optStrings.selectors.modalClose).click(); document.querySelector(optStrings.selectors.modalClose).click();
finishCallback(videoArray); finishCallback(videoArray);
@ -84,15 +82,15 @@ buldVidUrlArray = function(finishCallback) {
}, 20); }, 20);
}; };
getCurrentModalVideo = function() { getCurrentModalVideo = function () {
var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer); var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src); var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value; var shareLink = window.location.href;
var caption = document.querySelector(optStrings.selectors.modalCaption).textContent; var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
var soundLink = document.querySelector(optStrings.selectors.modalSoundLink); var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent; var uploader = document.querySelector(optStrings.selectors.modalUploader).textContent;
var soundHref = soundLink.getAttribute("href"); var soundHref = soundLink ? soundLink.getAttribute("href") : '';
var soundText = soundLink.text; var soundText = soundLink ? soundLink.text : '';
return { return {
url: vidUrl, url: vidUrl,
@ -106,16 +104,17 @@ getCurrentModalVideo = function() {
}; };
}; };
getCurrentVideo = function() { getCurrentVideo = function () {
if(checkForErrors()) return; //debugger;
if (checkForErrors()) return;
var player = document.querySelector(optStrings.selectors.videoPlayer); var player = document.querySelector(optStrings.selectors.videoPlayer);
var vidUrl = player.getAttribute(optStrings.attributes.src); var vidUrl = player.getAttribute(optStrings.attributes.src);
var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value; var shareLink = window.location.href;
var caption = document.querySelector(optStrings.selectors.videoCaption).textContent; var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
var soundLink = document.querySelector(optStrings.selectors.videoSoundLink); var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent; var uploader = document.querySelector(optStrings.selectors.videoUploader).textContent;
var soundHref = soundLink.getAttribute("href"); var soundHref = soundLink ? soundLink.getAttribute("href") : '';
var soundText = soundLink.text; var soundText = soundLink ? soundLink.text : '';
return { return {
url: vidUrl, url: vidUrl,
@ -131,19 +130,21 @@ getCurrentVideo = function() {
scrollBottom = () => window.scrollTo(0, document.body.scrollHeight); scrollBottom = () => window.scrollTo(0, document.body.scrollHeight);
scrollWhileNew = function(finishCallback) { scrollWhileNew = function (finishCallback) {
var state = { count: 0 }; var state = {
count: 0
};
var intervalID = window.setInterval(x => { var intervalID = window.setInterval(x => {
scrollBottom(); scrollBottom();
var oldCount = state.count; var oldCount = state.count;
state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length; state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
if(currentState.limit > 0) { if (currentState.limit > 0) {
if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) { if (currentState.preloadCount >= currentState.limit || state.count >= currentState.limit) {
finishCallback(createVidUrlElement); finishCallback(createVidUrlElement);
window.clearInterval(intervalID); window.clearInterval(intervalID);
} }
} }
if(checkForErrors()) { if (checkForErrors()) {
window.clearInterval(intervalID); window.clearInterval(intervalID);
return; return;
} else if (state.count == 0) { } else if (state.count == 0) {
@ -152,7 +153,7 @@ scrollWhileNew = function(finishCallback) {
if (oldCount !== state.count) { if (oldCount !== state.count) {
currentState.preloadCount = state.count; currentState.preloadCount = state.count;
} else { } else {
if (document.querySelector(optStrings.selectors.feedLoading)) { if (isLoading()) {
return; return;
} }
window.clearInterval(intervalID); window.clearInterval(intervalID);
@ -161,13 +162,18 @@ scrollWhileNew = function(finishCallback) {
}, 1000); }, 1000);
}; };
bootstrapIteratingVideos = function(limit) { isLoading = function () {
var loadingElement = document.querySelector(optStrings.selectors.feedLoading);
return loadingElement && loadingElement.getClientRects().length != 0;
}
bootstrapIteratingVideos = function (limit) {
currentState.limit = limit; currentState.limit = limit;
scrollWhileNew(buldVidUrlArray); scrollWhileNew(buldVidUrlArray);
return 'bootstrapIteratingVideos'; return 'bootstrapIteratingVideos';
}; };
bootstrapGetCurrentVideo = function() { bootstrapGetCurrentVideo = function () {
var video = getCurrentVideo(); var video = getCurrentVideo();
createVidUrlElement(video); createVidUrlElement(video);
return 'bootstrapGetCurrentVideo'; return 'bootstrapGetCurrentVideo';

View File

@ -0,0 +1,39 @@
package workflows
import (
"fmt"
models "github.com/pikami/tiktok-dl/models"
config "github.com/pikami/tiktok-dl/models/config"
res "github.com/pikami/tiktok-dl/resources"
utils "github.com/pikami/tiktok-dl/utils"
fileio "github.com/pikami/tiktok-dl/utils/fileio"
log "github.com/pikami/tiktok-dl/utils/log"
)
// CanUseDownloadScrapedData - Check's if DownloadScrapedData can be used
func CanUseDownloadScrapedData(scrapedDataFilePath string) bool {
return scrapedDataFilePath != ""
}
// DownloadScrapedData - Download items from scraped data file
func DownloadScrapedData(scrapedDataFilePath string) {
if !fileio.CheckIfExists(scrapedDataFilePath) {
log.LogFatal(res.ErrorPathNotFound, scrapedDataFilePath)
}
dataFileContent := fileio.ReadFileToString(scrapedDataFilePath)
uploads := models.ParseUploads(dataFileContent)
uploads = utils.RemoveArchivedItems(uploads)
uploadCount := len(uploads)
for index, upload := range uploads {
username := utils.GetUsernameFromString(upload.Uploader)
downloadDir := fmt.Sprintf("%s/%s", config.Config.OutputPath, username)
fileio.InitOutputDirectory(downloadDir)
downloadVideo(upload, downloadDir)
log.Logf(res.Downloaded, index+1, uploadCount)
}
log.Log()
}