Initial commit

fileio/downloader.go

@@ -0,0 +1,31 @@
+package fileio
+
+import (
+	"io"
+	"net/http"
+	"os"
+)
+
+// DownloadFile - Download file and store it
+func DownloadFile(outputFilename string, url string) {
+	// Get the data
+	resp, err := http.Get(url)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	// Create the file
+	out, err := os.Create(outputFilename)
+	if err != nil {
+		panic(err)
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+
+	if err != nil {
+		panic(err)
+	}
+}

fileio/file_rw.go

@@ -0,0 +1,47 @@
+package fileio
+
+import (
+	"fmt"
+	"os"
+)
+
+// WriteToFile - Writes a string to file
+func WriteToFile(filename string, content string) {
+	f, err := os.Create(filename)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	l, err := f.WriteString(content)
+	if err != nil {
+		fmt.Println(err)
+		f.Close()
+		return
+	}
+	fmt.Println(l, "bytes written successfully")
+	err = f.Close()
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+}
+
+// InitOutputDirectory - Creates output directory
+func InitOutputDirectory(path string) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		os.Mkdir(path, os.ModePerm)
+	}
+}
+
+// CheckIfExists - Checks if file or directory exists
+func CheckIfExists(path string) bool {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+
+	if _, err := os.Stat(path); !os.IsNotExist(err) {
+		return false
+	}
+
+	return true
+}

helpers/string_methods.go

@@ -0,0 +1,15 @@
+package helpers
+
+import (
+	"log"
+	"regexp"
+)
+
+// ToCleanString - replaces spaces with underscores
+func ToCleanString(str string) string {
+	reg, err := regexp.Compile("[^A-Za-z0-9]+")
+	if err != nil {
+		log.Fatal(err)
+	}
+	return reg.ReplaceAllString(str, "_")
+}

main.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "./fileio"
+	. "./helpers"
+	. "./structs"
+	"github.com/mmcdole/gofeed"
+)
+
+func main() {
+	args := os.Args[1:]
+	if len(args) < 1 {
+		fmt.Println("Usage: rss-dl [FEED_URL]")
+		return
+	}
+
+	fp := gofeed.NewParser()
+	feed, _ := fp.ParseURL(args[0])
+
+	outputDir := ToCleanString(feed.Title)
+	InitOutputDirectory(outputDir)
+
+	WriteToFile(outputDir+"/feed_details.json", GrabFeedDetailsJSON(feed))
+	for _, item := range feed.Items {
+		itemOutputFilename := ToCleanString(
+			item.PublishedParsed.Format("20060102") + "_" + item.Title)
+		itemOutputDir := outputDir + "/" + itemOutputFilename
+
+		if CheckIfExists(itemOutputDir) {
+			continue
+		}
+
+		InitOutputDirectory(itemOutputDir)
+		WriteToFile(
+			itemOutputDir+"/details.json",
+			GrabFeedItemJSON(item))
+		DownloadFile(
+			itemOutputDir+"/image"+filepath.Ext(item.Image.URL),
+			item.Image.URL)
+		for _, enclosure := range item.Enclosures {
+			DownloadFile(
+				itemOutputDir+"/"+filepath.Base(enclosure.URL),
+				enclosure.URL)
+		}
+	}
+}
+
+// GrabFeedDetailsJSON - Returns a feed summary json sring
+func GrabFeedDetailsJSON(feed *gofeed.Feed) string {
+	feedParsed := &FeedDetails{
+		Title:       feed.Title,
+		Description: feed.Description,
+		Categories:  feed.Categories,
+		Language:    feed.Language,
+		Link:        feed.Link,
+		Updated:     feed.Updated,
+	}
+
+	feedDetails, _ := json.Marshal(feedParsed)
+	return string(feedDetails)
+}
+
+// GrabFeedItemJSON - Returns a feed summary json sring
+func GrabFeedItemJSON(item *gofeed.Item) string {
+	itemParsed := &ItemDetails{
+		Title:       item.Title,
+		Description: item.Description,
+		Content:     item.Content,
+		Link:        item.Link,
+		Updated:     item.Updated,
+		Published:   item.Published,
+		GUID:        item.GUID,
+		Categories:  item.Categories,
+	}
+
+	itemDetails, _ := json.Marshal(itemParsed)
+	return string(itemDetails)
+}

structs/feed_details.go

@@ -0,0 +1,11 @@
+package structs
+
+// FeedDetails - Struct for outputing feed details summary
+type FeedDetails struct {
+	Title       string
+	Description string
+	Link        string
+	Updated     string
+	Language    string
+	Categories  []string
+}

structs/item_details.go

@@ -0,0 +1,13 @@
+package structs
+
+// ItemDetails - Struct for outputing item details summary
+type ItemDetails struct {
+	Title       string
+	Description string
+	Content     string
+	Link        string
+	Updated     string
+	Published   string
+	GUID        string
+	Categories  []string
+}