commit 07a977f4c882b1100d208961d41e8731b8d2ae3f Author: Pijus Kamandulis Date: Wed Apr 17 22:26:44 2019 +0300 Initial commit diff --git a/fileio/downloader.go b/fileio/downloader.go new file mode 100644 index 0000000..82a56d9 --- /dev/null +++ b/fileio/downloader.go @@ -0,0 +1,31 @@ +package fileio + +import ( + "io" + "net/http" + "os" +) + +// DownloadFile - Download file and store it +func DownloadFile(outputFilename string, url string) { + // Get the data + resp, err := http.Get(url) + if err != nil { + panic(err) + } + defer resp.Body.Close() + + // Create the file + out, err := os.Create(outputFilename) + if err != nil { + panic(err) + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + + if err != nil { + panic(err) + } +} diff --git a/fileio/file_rw.go b/fileio/file_rw.go new file mode 100644 index 0000000..9584fb8 --- /dev/null +++ b/fileio/file_rw.go @@ -0,0 +1,47 @@ +package fileio + +import ( + "fmt" + "os" +) + +// WriteToFile - Writes a string to file +func WriteToFile(filename string, content string) { + f, err := os.Create(filename) + if err != nil { + fmt.Println(err) + return + } + l, err := f.WriteString(content) + if err != nil { + fmt.Println(err) + f.Close() + return + } + fmt.Println(l, "bytes written successfully") + err = f.Close() + if err != nil { + fmt.Println(err) + return + } +} + +// InitOutputDirectory - Creates output directory +func InitOutputDirectory(path string) { + if _, err := os.Stat(path); os.IsNotExist(err) { + os.Mkdir(path, os.ModePerm) + } +} + +// CheckIfExists - Checks if file or directory exists +func CheckIfExists(path string) bool { + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + + if _, err := os.Stat(path); !os.IsNotExist(err) { + return false + } + + return true +} diff --git a/helpers/string_methods.go b/helpers/string_methods.go new file mode 100644 index 0000000..3d66414 --- /dev/null +++ b/helpers/string_methods.go @@ -0,0 +1,15 @@ +package helpers + +import ( + "log" + "regexp" +) + +// ToCleanString - replaces spaces with underscores +func ToCleanString(str string) string { + reg, err := regexp.Compile("[^A-Za-z0-9]+") + if err != nil { + log.Fatal(err) + } + return reg.ReplaceAllString(str, "_") +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..94e3d18 --- /dev/null +++ b/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + . "./fileio" + . "./helpers" + . "./structs" + "github.com/mmcdole/gofeed" +) + +func main() { + args := os.Args[1:] + if len(args) < 1 { + fmt.Println("Usage: rss-dl [FEED_URL]") + return + } + + fp := gofeed.NewParser() + feed, _ := fp.ParseURL(args[0]) + + outputDir := ToCleanString(feed.Title) + InitOutputDirectory(outputDir) + + WriteToFile(outputDir+"/feed_details.json", GrabFeedDetailsJSON(feed)) + for _, item := range feed.Items { + itemOutputFilename := ToCleanString( + item.PublishedParsed.Format("20060102") + "_" + item.Title) + itemOutputDir := outputDir + "/" + itemOutputFilename + + if CheckIfExists(itemOutputDir) { + continue + } + + InitOutputDirectory(itemOutputDir) + WriteToFile( + itemOutputDir+"/details.json", + GrabFeedItemJSON(item)) + DownloadFile( + itemOutputDir+"/image"+filepath.Ext(item.Image.URL), + item.Image.URL) + for _, enclosure := range item.Enclosures { + DownloadFile( + itemOutputDir+"/"+filepath.Base(enclosure.URL), + enclosure.URL) + } + } +} + +// GrabFeedDetailsJSON - Returns a feed summary json sring +func GrabFeedDetailsJSON(feed *gofeed.Feed) string { + feedParsed := &FeedDetails{ + Title: feed.Title, + Description: feed.Description, + Categories: feed.Categories, + Language: feed.Language, + Link: feed.Link, + Updated: feed.Updated, + } + + feedDetails, _ := json.Marshal(feedParsed) + return string(feedDetails) +} + +// GrabFeedItemJSON - Returns a feed summary json sring +func GrabFeedItemJSON(item *gofeed.Item) string { + itemParsed := &ItemDetails{ + Title: item.Title, + Description: item.Description, + Content: item.Content, + Link: item.Link, + Updated: item.Updated, + Published: item.Published, + GUID: item.GUID, + Categories: item.Categories, + } + + itemDetails, _ := json.Marshal(itemParsed) + return string(itemDetails) +} diff --git a/structs/feed_details.go b/structs/feed_details.go new file mode 100644 index 0000000..37a8bae --- /dev/null +++ b/structs/feed_details.go @@ -0,0 +1,11 @@ +package structs + +// FeedDetails - Struct for outputing feed details summary +type FeedDetails struct { + Title string + Description string + Link string + Updated string + Language string + Categories []string +} diff --git a/structs/item_details.go b/structs/item_details.go new file mode 100644 index 0000000..5a64fbb --- /dev/null +++ b/structs/item_details.go @@ -0,0 +1,13 @@ +package structs + +// ItemDetails - Struct for outputing item details summary +type ItemDetails struct { + Title string + Description string + Content string + Link string + Updated string + Published string + GUID string + Categories []string +}