TTDL-7 Added flag; Code clean up

This commit is contained in:
Pijus Kamandulis
2020-03-22 02:10:24 +02:00
parent 9a65746fd4
commit f9d35e3bf2
19 changed files with 188 additions and 79 deletions

View File

@@ -3,15 +3,16 @@ package client
import (
"context"
"errors"
"github.com/chromedp/chromedp"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/chromedp/chromedp"
config "../models/config"
utils "../utils"
log "../utils/log"
)
// GetMusicUploads - Get all uploads by given music
@@ -33,7 +34,7 @@ func executeClientAction(url string, jsAction string) (string, error) {
ctx, cancel := chromedp.NewContext(
allocCtx,
chromedp.WithLogf(log.Printf),
chromedp.WithLogf(log.Logf),
)
defer cancel()
@@ -84,9 +85,9 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
}
if jsOutput != "0" {
utils.Logf("\rPreloading... %s items have been founded.", jsOutput)
log.Logf("\rPreloading... %s items have been found.", jsOutput)
} else {
utils.Logf("\rPreloading...")
log.Logf("\rPreloading...")
}
if err := chromedp.Run(ctx, chromedp.EvaluateAsDevTools("currentState.finished.toString()", &jsOutput)); err != nil {
@@ -100,7 +101,7 @@ func runScrapeWithInfo(ctx context.Context, jsAction string, url string) (string
time.Sleep(50 * time.Millisecond)
}
utils.Log("\nRetrieving items...")
log.Log("\nRetrieving items...")
if err := chromedp.Run(ctx,
// Wait until custom js finishes
chromedp.WaitVisible(`video_urls`),

View File

@@ -1,22 +1,24 @@
package client
import (
"fmt"
models "../models"
config "../models/config"
"fmt"
)
// GetUserUploads - Get all uploads marked with given hashtag
// GetHashtagUploads - Get all uploads marked with given hashtag
func GetHashtagUploads(hashtagURL string) ([]models.Upload, error) {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput, err := executeClientAction(hashtagURL, jsMethod)
actionOutput, err := GetHashtagUploadsJSON(hashtagURL)
if err != nil {
return nil, err
}
return models.ParseUploads(actionOutput), nil
}
func GetHashtagUploadsJson(hashtagURL string) (string, error) {
// GetHashtagUploadsJSON - Get hashtag uploads scrape
func GetHashtagUploadsJSON(hashtagURL string) (string, error) {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput, err := executeClientAction(hashtagURL, jsMethod)
if err != nil {

View File

@@ -1,22 +1,23 @@
package client
import (
"fmt"
models "../models"
config "../models/config"
"fmt"
)
// GetMusicUploads - Get all uploads by given music
func GetMusicUploads(url string) ([]models.Upload, error) {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput, err := executeClientAction(url, jsMethod)
actionOutput, err := GetMusicUploadsJSON(url)
if err != nil {
return nil, err
}
return models.ParseUploads(actionOutput), nil
}
func GetMusicUploadsJson(url string) (string, error) {
// GetMusicUploadsJSON - Get music uploads scrape
func GetMusicUploadsJSON(url string) (string, error) {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput, err := executeClientAction(url, jsMethod)
if err != nil {

View File

@@ -1,22 +1,23 @@
package client
import (
"fmt"
models "../models"
config "../models/config"
"fmt"
)
// GetUserUploads - Get all uploads by user
func GetUserUploads(username string) ([]models.Upload, error) {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
actionOutput, err := GetUserUploadsJSON(username)
if err != nil {
return nil, err
}
return models.ParseUploads(actionOutput), nil
}
func GetUserUploadsJson(username string) (string, error) {
// GetUserUploadsJSON - Get user uploads scrape
func GetUserUploadsJSON(username string) (string, error) {
jsMethod := fmt.Sprintf("bootstrapIteratingVideos(%d)", config.Config.Limit)
actionOutput, err := executeClientAction(`https://www.tiktok.com/@`+username, jsMethod)
if err != nil {