From d3cf54472a9e4dc3487e007681e8bcbf2a7aa91f Mon Sep 17 00:00:00 2001 From: Pijus Kamandulis Date: Tue, 8 Jun 2021 19:56:04 +0300 Subject: [PATCH] Use go modules --- go.mod | 5 +++++ go.sum | 40 ++++++++++++++++++++++++++++++++++++++++ main.go | 53 +++++++++++++++++++++++++++-------------------------- 3 files changed, 72 insertions(+), 26 deletions(-) create mode 100644 go.mod create mode 100644 go.sum diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b1f887d --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/pikami/rss-dl + +go 1.16 + +require github.com/mmcdole/gofeed v1.1.3 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b62f0ab --- /dev/null +++ b/go.sum @@ -0,0 +1,40 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= +github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index ac8f977..af62970 100644 --- a/main.go +++ b/main.go @@ -5,60 +5,61 @@ import ( "fmt" "path/filepath" - . "./fileio" - . "./helpers" - . "./structs" "github.com/mmcdole/gofeed" + + fileio "github.com/pikami/rss-dl/fileio" + helpers "github.com/pikami/rss-dl/helpers" + structs "github.com/pikami/rss-dl/structs" ) func main() { - GetConfig() + structs.GetConfig() fp := gofeed.NewParser() - LogInfo("Downloading " + Config.FeedURL) - feed, _ := fp.ParseURL(Config.FeedURL) + helpers.LogInfo("Downloading " + structs.Config.FeedURL) + feed, _ := fp.ParseURL(structs.Config.FeedURL) - outputDir := Config.OutputPath + "/" + ToCleanString(feed.Title) - InitOutputDirectory(outputDir) + outputDir := structs.Config.OutputPath + "/" + helpers.ToCleanString(feed.Title) + fileio.InitOutputDirectory(outputDir) feedInfoPath := outputDir + "/feed_details.json" - LogInfo("Writing feed details as JSON to " + feedInfoPath) - WriteToFile(feedInfoPath, GrabFeedDetailsJSON(feed)) + helpers.LogInfo("Writing feed details as JSON to " + feedInfoPath) + fileio.WriteToFile(feedInfoPath, GrabFeedDetailsJSON(feed)) - feedImagePath := outputDir + "/image" + RemoveGetParams(filepath.Ext(feed.Image.URL)) - DownloadFile(feedImagePath, feed.Image.URL) + feedImagePath := outputDir + "/image" + helpers.RemoveGetParams(filepath.Ext(feed.Image.URL)) + fileio.DownloadFile(feedImagePath, feed.Image.URL) for _, item := range feed.Items { - itemOutputFilename := ToCleanString( + itemOutputFilename := helpers.ToCleanString( item.PublishedParsed.Format("20060102") + "_" + item.Title) itemOutputDir := outputDir + "/" + itemOutputFilename - if CheckIfExists(itemOutputDir) { + if fileio.CheckIfExists(itemOutputDir) { fmt.Println("Item '" + item.Title + "' already downloaded, skipping") continue } - LogInfo("Downloading feed item '" + item.Title + "' to " + itemOutputDir) - InitOutputDirectory(itemOutputDir) + helpers.LogInfo("Downloading feed item '" + item.Title + "' to " + itemOutputDir) + fileio.InitOutputDirectory(itemOutputDir) itemDetailsPath := itemOutputDir + "/details.json" - LogInfo("Writing details to " + itemDetailsPath) - WriteToFile( + helpers.LogInfo("Writing details to " + itemDetailsPath) + fileio.WriteToFile( itemDetailsPath, GrabFeedItemJSON(item)) if item.Image != nil { - itemImagePath := itemOutputDir + "/image" + RemoveGetParams(filepath.Ext(item.Image.URL)) - LogInfo("Downloading image to " + itemImagePath) - DownloadFile( + itemImagePath := itemOutputDir + "/image" + helpers.RemoveGetParams(filepath.Ext(item.Image.URL)) + helpers.LogInfo("Downloading image to " + itemImagePath) + fileio.DownloadFile( itemImagePath, item.Image.URL) } for _, enclosure := range item.Enclosures { - filename := RemoveGetParams(filepath.Base(enclosure.URL)) - LogInfo("Downloading attachment '" + filename + "'") - DownloadFile( + filename := helpers.RemoveGetParams(filepath.Base(enclosure.URL)) + helpers.LogInfo("Downloading attachment '" + filename + "'") + fileio.DownloadFile( itemOutputDir+"/"+filename, enclosure.URL) } @@ -67,7 +68,7 @@ func main() { // GrabFeedDetailsJSON - Returns a feed summary json sring func GrabFeedDetailsJSON(feed *gofeed.Feed) string { - feedParsed := &FeedDetails{ + feedParsed := &structs.FeedDetails{ Title: feed.Title, Description: feed.Description, Categories: feed.Categories, @@ -83,7 +84,7 @@ func GrabFeedDetailsJSON(feed *gofeed.Feed) string { // GrabFeedItemJSON - Returns a feed summary json sring func GrabFeedItemJSON(item *gofeed.Item) string { - itemParsed := &ItemDetails{ + itemParsed := &structs.ItemDetails{ Title: item.Title, Description: item.Description, Content: item.Content,