Embed scraper into compiled binary, extract strings to resources file

2026-06-12 23:37:08 +01:00 · 2020-04-12 16:56:50 +03:00
parent 70c3faf17e
commit ea2b866f9a
17 changed files with 370 additions and 42 deletions
@@ -0,0 +1,64 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	checkErr "github.com/pikami/tiktok-dl/utils/checkErr"
+	fileio "github.com/pikami/tiktok-dl/utils/fileio"
+)
+
+type resource struct {
+	Package  string
+	FileName string
+	Values   map[string]string
+}
+
+func (r resource) generate() {
+	filename := fmt.Sprintf("%s/%s", outputDir, r.FileName)
+	f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	checkErr.CheckErr(err)
+	defer f.Close()
+
+	// Header
+	header := fmt.Sprintf("// Package %s - This file is automatically generated.\n"+
+		"// Do not edit this file manually.\n"+
+		"// Check `/generator/resources.go` to change generated content\n"+
+		"package %s\n", r.Package, r.Package)
+
+	if _, err := f.WriteString(header); err != nil {
+		checkErr.CheckErr(err)
+	}
+
+	// Values
+	for key, value := range r.Values {
+		value = strings.ReplaceAll(value, "\n", "\\n")
+		value = strings.ReplaceAll(value, "\r", "\\r")
+		valueLine := fmt.Sprintf("\n//%s -\nvar %s = \"%s\"\n", key, key, value)
+		if _, err := f.WriteString(valueLine); err != nil {
+			checkErr.CheckErr(err)
+		}
+	}
+}
+
+func fileContentsOrDefault(file string) string {
+	defer func() {
+		if r := recover(); r != nil {
+			fmt.Printf("Failed to load file: %s\n", file)
+		}
+	}()
+
+	return safeString(fileio.ReadFileToString(file))
+}
+
+func safeString(str string) string {
+	escaped := strings.ReplaceAll(str, "\"", "\\\"")
+	return strings.ReplaceAll(escaped, "\n", "")
+}
+
+func main() {
+	for _, i := range res {
+		i.generate()
+	}
+}
@@ -0,0 +1,87 @@
+package main
+
+var (
+	outputDir = "../resources"
+
+	res = []resource{
+		resource{
+			Package:  "resources",
+			FileName: "scraper.go",
+			Values: map[string]string{
+				"ScraperPath":   "scraper.js",
+				"ScraperScript": fileContentsOrDefault("../scraper.min.js"),
+			},
+		},
+		resource{
+			Package:  "resources",
+			FileName: "errorStrings.go",
+			Values: map[string]string{
+				"ErrorCouldNotSerializeJSON": "Could not serialize json for video: %s\n",
+				"ErrorCouldNotRecogniseURL":  "Could not recognise URL format of string %s",
+				"Error":                      "Error : %s\n",
+				"ErrorPathNotFound":          "File path %s not found.",
+				"FailedOnItem":               "Failed while scraping item: %s\n",
+				"FailedToLoadScraper":        "Failed to load scraper",
+			},
+		},
+		resource{
+			Package:  "resources",
+			FileName: "messages.go",
+			Values: map[string]string{
+				"PreloadingItemsFound": "\rPreloading... %s items have been found.",
+				"Preloading":           "\rPreloading...",
+				"Retrieving":           "\nRetrieving items...",
+				"ItemsFoundInArchive":  "%d items, found in archive. Skipping...\n",
+				"Downloaded":           "\r[%d/%d] Downloaded",
+				"UsageLine": "Usage: tiktok-dl [OPTIONS] TIKTOK_USERNAME|TIKTOK_URL\n" +
+					"  or:  tiktok-dl [OPTIONS] -batch-file path/to/users.txt",
+			},
+		},
+		resource{
+			Package:  "resources",
+			FileName: "flags.go",
+			Values: map[string]string{
+				// Output
+				"OutputFlag":        "output",
+				"OutputDefault":     "./downloads",
+				"OutputDescription": "Output path",
+				// Batch file
+				"BatchFlag":        "batch-file",
+				"BatchDefault":     "",
+				"BatchDescription": "File containing URLs/Usernames to download, one value per line. Lines starting with '#', are considered as comments and ignored.",
+				// Archive
+				"ArchiveFlag":        "archive",
+				"ArchiveDefault":     "",
+				"ArchiveDescription": "Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.",
+				// Fail log
+				"FailLogFlag":        "fail-log",
+				"FailLogDefault":     "",
+				"FailLogDescription": "Write failed items to log file",
+				// Debug
+				"DebugFlag":        "debug",
+				"DebugDefault":     "false",
+				"DebugDescription": "Enables debug mode",
+				// Metadata
+				"MetadataFlag":        "metadata",
+				"MetadataDefault":     "false",
+				"MetadataDescription": "Write video metadata to a .json file",
+				// Quiet
+				"QuietFlag":        "quiet",
+				"QuietDefault":     "false",
+				"QuietDescription": "Suppress output",
+				// JSON only
+				"JsonFlag":        "json",
+				"JsonDefault":     "false",
+				"JsonDescription": "Just get JSON data from scraper (without video downloading)",
+				// Deadline
+				"DeadlineFlag":        "deadline",
+				"DeadlineDefault":     "1500",
+				"DeadlineDescription": "Sets the timout for scraper logic in seconds (used as a workaround for 'context deadline exceeded' error)",
+				// Limit
+				"LimitFlag":        "limit",
+				"LimitDefault":     "0",
+				"LimitDescription": "Sets the videos count limit (useful when there too many videos from the user or by hashtag)",
+			},
+		},
+	}
+)