Update build script to minimize js

.github/workflows/go.yml

@@ -15,30 +15,35 @@ jobs:
         go-version: 1.13
       id: go
 
+    - name: Set up node 10
+      uses: actions/setup-node@v1
+      with:
+        node-version: '10.x'
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v1
 
     - name: Get dependencies
       run: |
-        go get -v -t -d ./...
+        npm install
+        npm run install-dependencies
+
+    - name: Run unit tests
+      run: npm run test
 
     - name: Build
-      run: go build -v .
-
-    - name: Run tests
-      run: |
-        go test -v ./models
+      run: npm run build:dist
 
     - name: Upload Unix Artifacts
       if: startsWith(matrix.os, 'ubuntu-')
       uses: actions/upload-artifact@v1
       with:
         name: tiktok-dl_linux
-        path: tiktok-dl
+        path: out
         
     - name: Upload Windows Artifacts
       if: startsWith(matrix.os, 'windows-')
       uses: actions/upload-artifact@v1
       with:
         name: tiktok-dl_win64
-        path: tiktok-dl.exe
+        path: out

.gitignore

@@ -1,4 +1,5 @@
 .vscode
+node_modules
 __debug_bin
 downloads
 *.exe

README.md

@@ -1,17 +1,21 @@
 # TikTok-DL
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/pikami/tiktok-dl)](https://goreportcard.com/report/github.com/pikami/tiktok-dl)
+![GitHub Workflow Status](https://img.shields.io/github/workflow/status/pikami/tiktok-dl/tiktok-dl_CI)
 
 A simple tiktok video downloader written in go
 
 ## Basic usage
-Clone this repository and run `go build` to build the executable.\
+Download the executable from `https://github.com/pikami/tiktok-dl/releases`\
 You can download all videos from user by running `./tiktok-dl [Options] TIKTOK_USERNAME`\
 You can download single video by running `./tiktok-dl [Options] VIDEO_URL`
 
+## Build instructions
+Clone this repository and run `go build` to build the executable.
+
 ## Available options
 * `-debug` - enables debug mode
 * `-output some_directory` - Output path (default "./downloads")
 
 ## Acknowledgments
-This software uses the chromedp for web scraping, it can be found here: https://github.com/chromedp/chromedp
+This software uses the chromedp for web scraping, it can be found here: https://github.com/chromedp/chromedp

main.go

@@ -42,6 +42,11 @@ func downloadVideo(upload models.Upload, downloadDir string) {
 
 	fmt.Println("Downloading upload item '" + uploadID + "' to " + downloadPath)
 	utils.DownloadFile(downloadPath, upload.URL)
+
+	if models.Config.MetaData {
+		metadataPath := fmt.Sprintf("%s/%s.json", downloadDir, uploadID)
+		upload.WriteToFile(metadataPath)
+	}
 }
 
 func downloadUser() {

models/config.go

@@ -11,12 +11,14 @@ var Config struct {
 	URL        string
 	OutputPath string
 	Debug      bool
+	MetaData   bool
 }
 
 // GetConfig - Returns Config object
 func GetConfig() {
 	outputPath := flag.String("output", "./downloads", "Output path")
-	debug := flag.Bool("debug", false, "enables debug mode")
+	debug := flag.Bool("debug", false, "Enables debug mode")
+	metadata := flag.Bool("metadata", false, "Write video metadata to a .json file")
 	flag.Parse()
 
 	args := flag.Args()
@@ -28,4 +30,5 @@ func GetConfig() {
 	Config.URL = flag.Args()[len(args)-1]
 	Config.OutputPath = *outputPath
 	Config.Debug = *debug
+	Config.MetaData = *metadata
 }

models/upload.go

@@ -2,13 +2,23 @@ package models
 
 import (
 	"encoding/json"
+	"fmt"
+	"os"
 	"strings"
 )
 
 // Upload - Upload object
 type Upload struct {
-	ShareLink string `json:"shareLink"`
 	URL       string `json:"url"`
+	ShareLink string `json:"shareLink"`
+	Caption   string `json:"caption"`
+	Sound     Sound  `json:"sound"`
+}
+
+// Sound - Sound object
+type Sound struct {
+	Title string `json:"title"`
+	Link  string `json:"link"`
 }
 
 // ParseUploads - Parses json uploads array
@@ -30,3 +40,26 @@ func (u Upload) GetUploadID() string {
 	parts := strings.Split(u.ShareLink, "/")
 	return parts[len(parts)-1]
 }
+
+// WriteToFile - Writes object to file
+func (u Upload) WriteToFile(outputPath string) {
+	bytes, err := json.Marshal(u)
+	if err != nil {
+		fmt.Printf("Could not serialize json for video: %s", u.GetUploadID())
+		fmt.Println()
+		panic(err)
+	}
+
+	// Create the file
+	out, err := os.Create(outputPath)
+	if err != nil {
+		panic(err)
+	}
+	defer out.Close()
+
+	// Write to file
+	_, err = out.Write(bytes)
+	if err != nil {
+		panic(err)
+	}
+}

package-lock.json

@@ -0,0 +1,42 @@
+{
+    "name": "tiktok-dl",
+    "version": "0.0.1",
+    "lockfileVersion": 1,
+    "requires": true,
+    "dependencies": {
+        "buffer-from": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+            "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
+        },
+        "commander": {
+            "version": "2.20.3",
+            "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+            "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="
+        },
+        "source-map": {
+            "version": "0.6.1",
+            "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+            "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
+        },
+        "source-map-support": {
+            "version": "0.5.16",
+            "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz",
+            "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==",
+            "requires": {
+                "buffer-from": "^1.0.0",
+                "source-map": "^0.6.0"
+            }
+        },
+        "terser": {
+            "version": "4.6.3",
+            "resolved": "https://registry.npmjs.org/terser/-/terser-4.6.3.tgz",
+            "integrity": "sha512-Lw+ieAXmY69d09IIc/yqeBqXpEQIpDGZqT34ui1QWXIUpR2RjbqEkT8X7Lgex19hslSqcWM5iMN2kM11eMsESQ==",
+            "requires": {
+                "commander": "^2.20.0",
+                "source-map": "~0.6.1",
+                "source-map-support": "~0.5.12"
+            }
+        }
+    }
+}

package.json

@@ -0,0 +1,16 @@
+{
+    "name": "tiktok-dl",
+    "version": "0.0.1",
+    "scripts": {
+        "install-dependencies": "go get -v -t -d ./...",
+        "test": "go test -v ./models",
+        "clean": "rm -rf out",
+        "build:scraper": "node node_modules/terser/bin/terser -c -m -- scraper.js > out/scraper.js",
+        "build:app": "go build -o out/ -v .",
+        "build:dist": "mkdir out && npm run build:app && npm run build:scraper",
+        "build": "go build -v ."
+    },
+    "dependencies": {
+        "terser": "^4.6.3"
+    }
+}

scraper.js

@@ -1,14 +1,19 @@
 optStrings = {
     selectors: {
-        feedVideoItem: 'video-feed-item-wrapper',
+        feedLoading: 'div.tiktok-loading.feed-loading',
         modalArrowLeft: 'div.video-card-modal > div > img.arrow-right',
         modalClose: '.video-card-modal > div > div.close',
         modalPlayer: 'div > div > main > div.video-card-modal > div > div.video-card-big > div.video-card-container > div > div > video',
         modalShareInput: '.copy-link-container > input',
+        modalCaption: 'div.video-card-big > div.content-container > div.video-meta-info > h1',
+        modalSoundLink: 'div.content-container > div.video-meta-info > h2.music-info > a',
         videoPlayer: 'div.video-card-container > div > div > video',
         videoShareInput: 'div.content-container.border > div.copy-link-container > input',
+        videoCaption: 'div.content-container.border > div.video-meta-info > h1',
+        videoSoundLink: 'div.content-container.border > div.video-meta-info > h2.music-info > a',
     },
     classes: {
+        feedVideoItem: 'video-feed-item-wrapper',
         modalCloseDisabled: 'disabled',
     },
     tags: {
@@ -27,7 +32,7 @@ createVidUrlElement = function(outputObj) {
 }
 
 buldVidUrlArray = function(finishCallback) {
-    var feedItem = document.getElementsByClassName(optStrings.selectors.feedVideoItem)[0];
+    var feedItem = document.getElementsByClassName(optStrings.classes.feedVideoItem)[0];
     feedItem.click();
 
     var videoArray = [];
@@ -42,17 +47,26 @@ buldVidUrlArray = function(finishCallback) {
         } else {
             arrowRight.click();
         }
-    }, 500);
+    }, 20);
 };
 
 getCurrentModalVideo = function() {
     var modalPlayer = document.querySelector(optStrings.selectors.modalPlayer);
     var vidUrl = modalPlayer.getAttribute(optStrings.attributes.src);
     var shareLink = document.querySelector(optStrings.selectors.modalShareInput).value;
+    var caption = document.querySelector(optStrings.selectors.modalCaption).textContent;
+    var soundLink = document.querySelector(optStrings.selectors.modalSoundLink);
+    var soundHref = soundLink.getAttribute("href");
+    var soundText = soundLink.text;
 
     return {
         url: vidUrl,
-        shareLink: shareLink
+        shareLink: shareLink,
+        caption: caption,
+        sound: {
+            title: soundText,
+            link: soundHref,
+        },
     };
 }
 
@@ -60,10 +74,19 @@ getCurrentVideo = function() {
     var player = document.querySelector(optStrings.selectors.videoPlayer);
     var vidUrl = player.getAttribute(optStrings.attributes.src);
     var shareLink = document.querySelector(optStrings.selectors.videoShareInput).value;
+    var caption = document.querySelector(optStrings.selectors.videoCaption).textContent;
+    var soundLink = document.querySelector(optStrings.selectors.videoSoundLink);
+    var soundHref = soundLink.getAttribute("href");
+    var soundText = soundLink.text;
 
     return {
         url: vidUrl,
-        shareLink: shareLink
+        shareLink: shareLink,
+        caption: caption,
+        sound: {
+            title: soundText,
+            link: soundHref,
+        },
     };
 }
 
@@ -71,24 +94,22 @@ scrollWhileNew = function(finishCallback) {
     var state = { count: 0 };
     var intervalID = window.setInterval(x => {
         var oldCount = state.count;
-        state.count = document.getElementsByClassName(optStrings.selectors.feedVideoItem).length;
+        state.count = document.getElementsByClassName(optStrings.classes.feedVideoItem).length;
         if (oldCount !== state.count) {
             window.scrollTo(0, document.body.scrollHeight);
         } else {
+            if (document.querySelector(optStrings.selectors.feedLoading)) {
+                window.scrollTo(0, document.body.scrollHeight);
+                return;
+            }
             window.clearInterval(intervalID);
-            finishCallback();
+            finishCallback(createVidUrlElement);
         }
     }, 1000);
 };
 
 bootstrapIteratingVideos = function() {
-    var intervalID = window.setInterval(() => {
-        window.scrollTo(0, document.body.scrollHeight);
-        if (document.getElementsByClassName(optStrings.selectors.feedVideoItem).length > 0) {
-            window.setTimeout(() => buldVidUrlArray(createVidUrlElement), 100);
-            window.clearInterval(intervalID);
-        }
-    }, 500);
+    scrollWhileNew(buldVidUrlArray);
     return 'bootstrapIteratingVideos';
 };
 
@@ -102,7 +123,7 @@ init = () => {
     const newProto = navigator.__proto__;
     delete newProto.webdriver;
     navigator.__proto__ = newProto;
+    return 'script initialized';
 };
 
-init();
-'script initialized'
+init();