mirror of https://github.com/pikami/rss-dl.git
Use go modules
This commit is contained in:
Pijus Kamandulis
parent
c130a4ef4d
commit
d3cf54472a
|
|
@ -0,0 +1,5 @@
|
|||
module github.com/pikami/rss-dl
|
||||
|
||||
go 1.16
|
||||
|
||||
require github.com/mmcdole/gofeed v1.1.3
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
|
||||
github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
|
||||
github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
|
||||
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
|
||||
github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
53
main.go
53
main.go
|
|
@ -5,60 +5,61 @@ import (
|
|||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
. "./fileio"
|
||||
. "./helpers"
|
||||
. "./structs"
|
||||
"github.com/mmcdole/gofeed"
|
||||
|
||||
fileio "github.com/pikami/rss-dl/fileio"
|
||||
helpers "github.com/pikami/rss-dl/helpers"
|
||||
structs "github.com/pikami/rss-dl/structs"
|
||||
)
|
||||
|
||||
func main() {
|
||||
GetConfig()
|
||||
structs.GetConfig()
|
||||
|
||||
fp := gofeed.NewParser()
|
||||
LogInfo("Downloading " + Config.FeedURL)
|
||||
feed, _ := fp.ParseURL(Config.FeedURL)
|
||||
helpers.LogInfo("Downloading " + structs.Config.FeedURL)
|
||||
feed, _ := fp.ParseURL(structs.Config.FeedURL)
|
||||
|
||||
outputDir := Config.OutputPath + "/" + ToCleanString(feed.Title)
|
||||
InitOutputDirectory(outputDir)
|
||||
outputDir := structs.Config.OutputPath + "/" + helpers.ToCleanString(feed.Title)
|
||||
fileio.InitOutputDirectory(outputDir)
|
||||
|
||||
feedInfoPath := outputDir + "/feed_details.json"
|
||||
LogInfo("Writing feed details as JSON to " + feedInfoPath)
|
||||
WriteToFile(feedInfoPath, GrabFeedDetailsJSON(feed))
|
||||
helpers.LogInfo("Writing feed details as JSON to " + feedInfoPath)
|
||||
fileio.WriteToFile(feedInfoPath, GrabFeedDetailsJSON(feed))
|
||||
|
||||
feedImagePath := outputDir + "/image" + RemoveGetParams(filepath.Ext(feed.Image.URL))
|
||||
DownloadFile(feedImagePath, feed.Image.URL)
|
||||
feedImagePath := outputDir + "/image" + helpers.RemoveGetParams(filepath.Ext(feed.Image.URL))
|
||||
fileio.DownloadFile(feedImagePath, feed.Image.URL)
|
||||
|
||||
for _, item := range feed.Items {
|
||||
itemOutputFilename := ToCleanString(
|
||||
itemOutputFilename := helpers.ToCleanString(
|
||||
item.PublishedParsed.Format("20060102") + "_" + item.Title)
|
||||
itemOutputDir := outputDir + "/" + itemOutputFilename
|
||||
|
||||
if CheckIfExists(itemOutputDir) {
|
||||
if fileio.CheckIfExists(itemOutputDir) {
|
||||
fmt.Println("Item '" + item.Title + "' already downloaded, skipping")
|
||||
continue
|
||||
}
|
||||
|
||||
LogInfo("Downloading feed item '" + item.Title + "' to " + itemOutputDir)
|
||||
InitOutputDirectory(itemOutputDir)
|
||||
helpers.LogInfo("Downloading feed item '" + item.Title + "' to " + itemOutputDir)
|
||||
fileio.InitOutputDirectory(itemOutputDir)
|
||||
|
||||
itemDetailsPath := itemOutputDir + "/details.json"
|
||||
LogInfo("Writing details to " + itemDetailsPath)
|
||||
WriteToFile(
|
||||
helpers.LogInfo("Writing details to " + itemDetailsPath)
|
||||
fileio.WriteToFile(
|
||||
itemDetailsPath,
|
||||
GrabFeedItemJSON(item))
|
||||
|
||||
if item.Image != nil {
|
||||
itemImagePath := itemOutputDir + "/image" + RemoveGetParams(filepath.Ext(item.Image.URL))
|
||||
LogInfo("Downloading image to " + itemImagePath)
|
||||
DownloadFile(
|
||||
itemImagePath := itemOutputDir + "/image" + helpers.RemoveGetParams(filepath.Ext(item.Image.URL))
|
||||
helpers.LogInfo("Downloading image to " + itemImagePath)
|
||||
fileio.DownloadFile(
|
||||
itemImagePath,
|
||||
item.Image.URL)
|
||||
}
|
||||
|
||||
for _, enclosure := range item.Enclosures {
|
||||
filename := RemoveGetParams(filepath.Base(enclosure.URL))
|
||||
LogInfo("Downloading attachment '" + filename + "'")
|
||||
DownloadFile(
|
||||
filename := helpers.RemoveGetParams(filepath.Base(enclosure.URL))
|
||||
helpers.LogInfo("Downloading attachment '" + filename + "'")
|
||||
fileio.DownloadFile(
|
||||
itemOutputDir+"/"+filename,
|
||||
enclosure.URL)
|
||||
}
|
||||
|
|
@ -67,7 +68,7 @@ func main() {
|
|||
|
||||
// GrabFeedDetailsJSON - Returns a feed summary json sring
|
||||
func GrabFeedDetailsJSON(feed *gofeed.Feed) string {
|
||||
feedParsed := &FeedDetails{
|
||||
feedParsed := &structs.FeedDetails{
|
||||
Title: feed.Title,
|
||||
Description: feed.Description,
|
||||
Categories: feed.Categories,
|
||||
|
|
@ -83,7 +84,7 @@ func GrabFeedDetailsJSON(feed *gofeed.Feed) string {
|
|||
|
||||
// GrabFeedItemJSON - Returns a feed summary json sring
|
||||
func GrabFeedItemJSON(item *gofeed.Item) string {
|
||||
itemParsed := &ItemDetails{
|
||||
itemParsed := &structs.ItemDetails{
|
||||
Title: item.Title,
|
||||
Description: item.Description,
|
||||
Content: item.Content,
|
||||
|
|
|
|||
Loading…
Reference in New Issue