rss-dl/main.go

99 lines
2.5 KiB
Go
Raw Normal View History

2019-04-17 20:26:44 +01:00
package main
import (
"encoding/json"
"fmt"
"path/filepath"
"strings"
2019-04-17 20:26:44 +01:00
. "./fileio"
. "./helpers"
. "./structs"
"github.com/mmcdole/gofeed"
)
func main() {
2019-04-20 20:30:42 +01:00
GetConfig()
2019-04-17 20:26:44 +01:00
fp := gofeed.NewParser()
2019-04-20 20:30:42 +01:00
LogInfo("Downloading " + Config.FeedURL)
feed, _ := fp.ParseURL(Config.FeedURL)
2019-04-17 20:26:44 +01:00
2019-04-20 20:30:42 +01:00
outputDir := Config.OutputPath + "/" + ToCleanString(feed.Title)
2019-04-17 20:26:44 +01:00
InitOutputDirectory(outputDir)
2019-04-20 19:48:28 +01:00
feedInfoPath := outputDir + "/feed_details.json"
LogInfo("Writing feed details as JSON to " + feedInfoPath)
WriteToFile(feedInfoPath, GrabFeedDetailsJSON(feed))
2019-05-05 18:06:09 +01:00
feedImagePath := outputDir + "/image" + filepath.Ext(feed.Image.URL)
DownloadFile(feedImagePath, feed.Image.URL)
2019-04-17 20:26:44 +01:00
for _, item := range feed.Items {
itemOutputFilename := ToCleanString(
item.PublishedParsed.Format("20060102") + "_" + item.Title)
itemOutputDir := outputDir + "/" + itemOutputFilename
if CheckIfExists(itemOutputDir) {
2019-04-20 19:48:28 +01:00
fmt.Println("Item '" + item.Title + "' already downloaded, skipping")
2019-04-17 20:26:44 +01:00
continue
}
2019-04-20 19:48:28 +01:00
LogInfo("Downloading feed item '" + item.Title + "' to " + itemOutputDir)
2019-04-17 20:26:44 +01:00
InitOutputDirectory(itemOutputDir)
2019-04-20 19:48:28 +01:00
itemDetailsPath := itemOutputDir + "/details.json"
LogInfo("Writing details to " + itemDetailsPath)
2019-04-17 20:26:44 +01:00
WriteToFile(
2019-04-20 19:48:28 +01:00
itemDetailsPath,
2019-04-17 20:26:44 +01:00
GrabFeedItemJSON(item))
2019-04-20 19:48:28 +01:00
itemImagePath := itemOutputDir + "/image" + filepath.Ext(item.Image.URL)
LogInfo("Downloading image to " + itemImagePath)
2019-04-17 20:26:44 +01:00
DownloadFile(
2019-04-20 19:48:28 +01:00
itemImagePath,
2019-04-17 20:26:44 +01:00
item.Image.URL)
2019-04-20 19:48:28 +01:00
2019-04-17 20:26:44 +01:00
for _, enclosure := range item.Enclosures {
filename := strings.Split(filepath.Base(enclosure.URL), "?")[0]
LogInfo("Downloading attachment '" + filename + "'")
2019-04-17 20:26:44 +01:00
DownloadFile(
itemOutputDir+"/"+filename,
2019-04-17 20:26:44 +01:00
enclosure.URL)
}
}
}
// GrabFeedDetailsJSON - Returns a feed summary json sring
func GrabFeedDetailsJSON(feed *gofeed.Feed) string {
feedParsed := &FeedDetails{
Title: feed.Title,
Description: feed.Description,
Categories: feed.Categories,
Language: feed.Language,
Link: feed.Link,
2019-05-05 18:06:09 +01:00
FeedLink: feed.FeedLink,
2019-04-17 20:26:44 +01:00
Updated: feed.Updated,
}
feedDetails, _ := json.Marshal(feedParsed)
return string(feedDetails)
}
// GrabFeedItemJSON - Returns a feed summary json sring
func GrabFeedItemJSON(item *gofeed.Item) string {
itemParsed := &ItemDetails{
Title: item.Title,
Description: item.Description,
Content: item.Content,
Link: item.Link,
Updated: item.Updated,
Published: item.Published,
GUID: item.GUID,
Categories: item.Categories,
}
itemDetails, _ := json.Marshal(itemParsed)
return string(itemDetails)
}